From d4d1495cf80ed7e7e8daed1c14fa0fb0cd643b05 Mon Sep 17 00:00:00 2001 From: TommyLike Date: Wed, 11 Sep 2019 17:40:33 +0800 Subject: [PATCH 1/3] Fix debug issue --- mail/dockerfile/core/assets/mailman.cfg | 3 + .../dockerfile/core_utils/Dockerfile.git_tool | 9 + .../core_utils/docker-entrypoint.sh | 19 ++ .../core_utils/mailman-core-utils.py | 165 ++++++++++++++++++ mail/openeuler.org/mailcore-utils.yaml | 78 +++++++++ mail/templates/README.md | 37 ++++ .../list-user-notice-welcome/community.txt | 29 +++ 7 files changed, 340 insertions(+) create mode 100644 mail/dockerfile/core_utils/Dockerfile.git_tool create mode 100755 mail/dockerfile/core_utils/docker-entrypoint.sh create mode 100644 mail/dockerfile/core_utils/mailman-core-utils.py create mode 100644 mail/openeuler.org/mailcore-utils.yaml create mode 100644 mail/templates/README.md create mode 100644 mail/templates/list-user-notice-welcome/community.txt diff --git a/mail/dockerfile/core/assets/mailman.cfg b/mail/dockerfile/core/assets/mailman.cfg index 8115aecd..e66159ca 100644 --- a/mail/dockerfile/core/assets/mailman.cfg +++ b/mail/dockerfile/core/assets/mailman.cfg @@ -33,3 +33,6 @@ configuration: /opt/mailman/mailman-hyperkitty.cfg [database] class: mailman.database.postgresql.PostgreSQLDatabase url: postgres://mailman:mailmanpass@database/mailmandb + +[mailman] +default_language: zh_CN \ No newline at end of file diff --git a/mail/dockerfile/core_utils/Dockerfile.git_tool b/mail/dockerfile/core_utils/Dockerfile.git_tool new file mode 100644 index 00000000..a66a3242 --- /dev/null +++ b/mail/dockerfile/core_utils/Dockerfile.git_tool @@ -0,0 +1,9 @@ +FROM alpine + +MAINTAINER TommyLike + +RUN apk --update add git less openssh && \ + rm -rf /var/lib/apt/lists/* && \ + rm /var/cache/apk/* + +ENTRYPOINT ["git"] diff --git a/mail/dockerfile/core_utils/docker-entrypoint.sh b/mail/dockerfile/core_utils/docker-entrypoint.sh new file mode 100755 index 00000000..363a7994 --- /dev/null +++ b/mail/dockerfile/core_utils/docker-entrypoint.sh @@ -0,0 +1,19 @@ +#! /bin/sh +set -e + + +function wait_for_mailcore_ready() { + if [[ -z "${MAILMAN_CORE_ENDPOINT}" ]]; then + echo "MAILMAN_CORE_ENDPOINT is not defined. exit none zero." + exit 1 + fi + echo "accessing mailcore endpoints ${MAILMAN_CORE_ENDPOINT}" + while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' $MAILMAN_CORE_ENDPOINT)" != "401" ]]; do + sleep 1; + echo "retrying: accessing mailcore endpoints ${MAILMAN_CORE_ENDPOINT}" + done +} + +wait_for_mailcore_ready + +exec $@ diff --git a/mail/dockerfile/core_utils/mailman-core-utils.py b/mail/dockerfile/core_utils/mailman-core-utils.py new file mode 100644 index 00000000..a64a2dbe --- /dev/null +++ b/mail/dockerfile/core_utils/mailman-core-utils.py @@ -0,0 +1,165 @@ +import requests +import os +import signal +from mailmanclient import Client +from http.server import SimpleHTTPRequestHandler, HTTPServer + + +# This file used to create the default domain, list and welcome templates +# for mail list +# the configuration are listed below: +MAILMAN_CORE_ENDPOINT = os.environ.get( + "MAILMAN_CORE_ENDPOINT", + 'http://mailman-core-0.mail-suit-service.default.svc.cluster.local:8001/3.1') + +MAILMAN_CORE_USER = os.environ.get("MAILMAN_CORE_USER", "restadmin") + +MAILMAN_CORE_PASSWORD = os.environ.get("MAILMAN_CORE_PASSWORD", "restpass") + +DEFAULT_DOMAIN_NAME = os.environ.get("DEFAULT_DOMAIN_NAME", "openeuler.org") + +DEFAULT_MAIL_LISTS = os.environ.get("DEFAULT_MAIL_LISTS", "dev,community,user") + +# configure used for http server for mailman core service +TEMPLATE_FOLDER_PATH = os.environ.get("TEMPLATE_FOLDER_PATH", "templates") +TEMPLATE_SERVER_ADDRESS = os.environ.get("TEMPLATE_SERVER_ADDRESS", + "127.0.0.1") +TEMPLATE_SERVER_PORT = os.environ.get("TEMPLATE_SERVER_PORT", 8000) + +TEMPLATE_FOLDER_CONVERSION_EXCEPTION = { + "domain-admin-notice-new-list": "domain:admin:notice:new-list", + "list-user-notice-no-more-today": "list:user:notice:no-more-today", +} + + +class TemplateHandler(SimpleHTTPRequestHandler): + + def do_GET(self): + # Allow access for templates folder. + if not str.lstrip(self.path, "/").startswith("templates"): + self.send_response(403) + self.send_header('Content-type', 'text/plain') + self.end_headers() + self.wfile.write( + bytes("Only resource under templates folder are accessible!", + 'UTF-8')) + else: + super(TemplateHandler, self).do_GET() + + +class SignalException(Exception): + pass + + +def prepare_list(): + # pre-check before handling mailman core service + if DEFAULT_DOMAIN_NAME == "": + print("Must specify 'DEFAULT_DOMAIN_NAME' for mail list preparation.") + exit(1) + + lists = str.split(str(DEFAULT_MAIL_LISTS).lower(), ",") + if not os.path.exists(TEMPLATE_FOLDER_PATH): + print("The template file folder 'DEFAULT_MAIL_LISTS' must exits on" + " local.") + exit(1) + + if len(lists) == 0: + # find out all of the lists from local folder. + local_file = [] + for _, _, f in os.walk(os.path.join(os.getcwd(), + TEMPLATE_FOLDER_PATH)): + for file in f: + if file.endswith(".txt"): + local_file.append(os.path.splitext(file)[0]) + lists = list(set(local_file)) + + client = Client(MAILMAN_CORE_ENDPOINT, + MAILMAN_CORE_USER, + MAILMAN_CORE_PASSWORD) + + # Create default domain if not exists + default_domain = client.get_domain(DEFAULT_DOMAIN_NAME) + if default_domain is None: + default_domain = client.create_domain(DEFAULT_DOMAIN_NAME) + + # Create default mail lists + existing_lists = [el.list_name for el in client.lists] + for l in lists: + if l in existing_lists: + print("skip creating list {0}, since it's already exist".format(l)) + continue + else: + print("starting to create mail list {0}".format(l)) + default_domain.create_list(l) + + # Patch template for lists + for l in lists: + # browse all of the dirs and find out the template files + existing_folders = [ + f for f in os.listdir( + os.path.join(os.getcwd(), TEMPLATE_FOLDER_PATH))] + for d in existing_folders: + # check the list file exists + local_file = get_template_file(d, l) + if os.path.exists(local_file): + patch_content = { + convert_name_to_substitution(d): get_templates_url(d, l) + } + patch_uri = "{0}/lists/{1}.{2}/uris".format( + MAILMAN_CORE_ENDPOINT, + l, + DEFAULT_DOMAIN_NAME) + response = requests.patch( + patch_uri, patch_content, + auth=(MAILMAN_CORE_USER, MAILMAN_CORE_PASSWORD)) + print("patching list {0} with template file {1}, result {2} {3}" + "".format(l, local_file, response.status_code, response.text)) + else: + print("could not found template file for list {0}, path {1}, " + "skipping".format(l, local_file)) + + +def convert_name_to_substitution(dir_name): + if dir_name in TEMPLATE_FOLDER_CONVERSION_EXCEPTION: + return TEMPLATE_FOLDER_CONVERSION_EXCEPTION[dir_name] + return str(dir_name).replace("-", ":") + + +def get_templates_url(dir_name, list_name): + return "http://{0}:{1}/templates/{2}/{3}.txt".format( + TEMPLATE_SERVER_ADDRESS, TEMPLATE_SERVER_PORT, dir_name, list_name) + + +def get_template_file(folder_name, list_name): + return os.path.join(os.getcwd(), + TEMPLATE_FOLDER_PATH, + folder_name, "{0}.txt".format(list_name)) + + +def httpd_signal_handler(signum, frame): + print("signal received {0}, exiting".format(signum)) + raise SignalException() + + +def running_templates_server(): + httpd = HTTPServer((TEMPLATE_SERVER_ADDRESS, int(TEMPLATE_SERVER_PORT)), + TemplateHandler) + # Force encoding to UTF-8 + m = SimpleHTTPRequestHandler.extensions_map + m[''] = 'text/plain' + m.update(dict([(k, v + ';charset=utf-8') for k, v in m.items()])) + print("template server starts at {0}:{1}".format(TEMPLATE_SERVER_ADDRESS, + TEMPLATE_SERVER_PORT)) + try: + # exit with 0 when sigterm signal received. + signal.signal(signal.SIGTERM, httpd_signal_handler) + httpd.serve_forever() + except (IndentationError, SignalException): + pass + print("template server ends") + httpd.server_close() + + +if __name__ == "__main__": + prepare_list() + running_templates_server() diff --git a/mail/openeuler.org/mailcore-utils.yaml b/mail/openeuler.org/mailcore-utils.yaml new file mode 100644 index 00000000..e4434115 --- /dev/null +++ b/mail/openeuler.org/mailcore-utils.yaml @@ -0,0 +1,78 @@ +# Deployment for mail core utils service +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: mailman-core-utils + namespace: default + labels: + component: mail-core-utils + app: mail-suit-service +spec: + replicas: 1 + selector: + matchLabels: + component: mail-core-utils + app: mail-suit-service + template: + metadata: + labels: + component: mail-core-utils + app: mail-suit-service + spec: + # init container used to clone the repo and move it into the dir that core-utils will use. + initContainers: + - name: mail-git-tools + # Please ensure the image exists + image: openeuler/git-tools:0.0.1 + imagePullPolicy: "IfNotPresent" + volumeMounts: + - mountPath: /opt/mailman-utils/data_source + name: mailman-core-utils + env: + # the repo which contains the templates folder + - name: TEMPLATE_REPO + value: https://gitee.com/openeuler/infrastructure.git + command: + - /bin/sh + - -c + - | + cd /opt/mailman-utils/data_source; + git clone ${TEMPLATE_REPO} + containers: + - name: mailman-core-utils + # Please ensure the image exists + image: openeuler/mailman-core-utils:0.0.1 + imagePullPolicy: "IfNotPresent" + volumeMounts: + - mountPath: /opt/mailman-utils/data_source + name: mailman-core-utils + env: + # the mailman core API used to handling creating list and replace templates + - name: MAILMAN_CORE_ENDPOINT + value: http://mailman-core-0.mail-suit-service.default.svc.cluster.local:8001/3.1 + # the credential for mailman core API + - name: MAILMAN_CORE_USER + value: restadmin + - name: MAILMAN_CORE_PASSWORD + value: restpass + # default domain name that will be created when starts up + - name: DEFAULT_DOMAIN_NAME + value: openeuler + # default mail lists that will created when starts up, if not specified, container will scan all of the list file in templates folder. + - name: DEFAULT_MAIL_LISTS + value: community,announce,council,dev,infra,marketing,user-committee + # the template server address, this is default to self IP address + - name: TEMPLATE_SERVER_ADDRESS + valueFrom: + fieldRef: + fieldPath: status.podIP + command: + - /bin/sh + - -c + - | + cp -r /opt/mailman-utils/data_source/infrastructure/mail/templates .; + exec docker-entrypoint.sh python -u mailman-core-utils.py + volumes: + - name: mailman-core-utils + emptyDir: {} diff --git a/mail/templates/README.md b/mail/templates/README.md new file mode 100644 index 00000000..06d84bf3 --- /dev/null +++ b/mail/templates/README.md @@ -0,0 +1,37 @@ +# README +**NOTE**: Please read the guidance below before you starting changing any content inside of this folder. + +## Introduce +This files inside of this folder are used for replacing the default mailman core email templates when the mailman suite +services starts up, therefore these files are strictly arranged by the design of mailman templates mechanism. + +There are many different template substitutions provided by mailman core service, please refer [here](https://mailman.readthedocs.io/en/latest/src/mailman/rest/docs/templates.html#templated-texts) +to have a better understanding on this. In short, mailman templates are classified with different keys, here below are some of +these template keys: +1. `domain:admin:notice:new-list`: Sent to the administrators of any newly created mailing list. +2. `list:user:action:subscribe`: The message sent to subscribers when a subscription confirmation is required. +3. `list:user:notice:welcome`: The notice sent to a member when they are subscribed to the mailing list. + +For instance, if we require to update the `welcome` templates of `developing` list (on domain `example.com`), we need to update the list uris via +mailman core API: +```python +import requests +requests.patch('http://{host}:{port}/3.1/lists/developing.example.com/uris', + {'list:user:notice:welcome': 'http://{http_path_which_store_template_file}'}, + auth=({username}, {password})) +``` + +## Folder Structure +Mailman's `core-utils` will help us to setting up the http server and invoke mailman's core API to patch the template, in order to +achieve this automatically, the structure of folder `templates` are arranged below: + +```$xslt +infrastructure +├─────────templates +│   ├───list-user-notice-welcome //the mail template key, all ":" will be replaced by "-" +│ │ ├───────developing.txt +│ │ └───────community.txt +│ └───domain-admin-notice-new-list +│ └───────user.txt //the template file which is named in the format of "{list name}.txt", Do not capitalize first letter. +``` +Once the content in templates folder have been updated, we can update the mailman templates through recreating the `core-utils` pods in cluster. diff --git a/mail/templates/list-user-notice-welcome/community.txt b/mail/templates/list-user-notice-welcome/community.txt new file mode 100644 index 00000000..4a499531 --- /dev/null +++ b/mail/templates/list-user-notice-welcome/community.txt @@ -0,0 +1,29 @@ +Email Address Registration Confirmation + +Hello, this is the GNU Mailman server at $domain. + +We have received a registration request for the email address + + $user_email + +Before you can start using GNU Mailman at this site, you must first confirm that this is your email address. You can do this by replying to this message, keeping the Subject header intact. + +If you do not wish to register this email address, simply disregard this message. If you think you are being maliciously subscribed to the list, or have any other questions, you may contact + + $owner_email + +邮件列表注册确认邮件。 + +你好！这是$domain邮件列表的自动回复邮件，请阅读完毕。 + +我们已经收到了如下邮件地址发送的注册请求： + + $user_email + +如果你确认如上地址是你的邮件地址，请回复该邮件，不要修改邮件标题和邮件内容，直接回复，之后你就可以使用该邮件列表了。 + +如果你不想订阅该邮件列表，请忽略该邮件。 + +如果你被恶意订阅了该邮件列表，或者你有关于该邮件列表的其他问题需要咨询，请发送邮件到给 + + $owner_email -- Gitee From 421981d071ee5dbb06e7299bcc1bced562e3557d Mon Sep 17 00:00:00 2001 From: TommyLike Date: Thu, 12 Sep 2019 13:17:06 +0800 Subject: [PATCH 2/3] Resolve conflict issues --- mail/dockerfile/core_utils/Dockerfile | 20 ++++++++++++++++++++ mail/dockerfile/core_utils/README.md | 11 +++++++++++ mail/openeuler.org/mailcore-utils.yaml | 2 +- 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 mail/dockerfile/core_utils/Dockerfile create mode 100644 mail/dockerfile/core_utils/README.md diff --git a/mail/dockerfile/core_utils/Dockerfile b/mail/dockerfile/core_utils/Dockerfile new file mode 100644 index 00000000..61016c69 --- /dev/null +++ b/mail/dockerfile/core_utils/Dockerfile @@ -0,0 +1,20 @@ +FROM frolvlad/alpine-python3:latest + +MAINTAINER TommyLike + +COPY mailman-core-utils.py /opt/mailman-utils/mailman-core-utils.py + +COPY docker-entrypoint.sh /usr/local/bin/ + +RUN apk add curl && \ + pip3 install requests && \ + pip3 install mailmanclient==3.3.0 && \ + rm -r /root/.cache + +EXPOSE 8000 + +ENTRYPOINT ["docker-entrypoint.sh"] + +WORKDIR /opt/mailman-utils + +CMD ["python3", "-u", "mailman-core-utils.py"] diff --git a/mail/dockerfile/core_utils/README.md b/mail/dockerfile/core_utils/README.md new file mode 100644 index 00000000..e70edc9b --- /dev/null +++ b/mail/dockerfile/core_utils/README.md @@ -0,0 +1,11 @@ +# Core util Dockerfile +These images are used to help mailman core service to setup the basic domain&list and replace +default templates with the file located in repo's `templates` folder. + +There are two different docker images are used here: +1. git-tools: This image used to clone the infrastructure repo from gitee and move templates folder + to the core utils image working folder, please check the dockerfile `Dockerfile.git_tool` + +2. core-utils: This image used to setting up the basic domain&lists and replace templates for mailman core service, + please check the dockerfile `Dockerfile`. + diff --git a/mail/openeuler.org/mailcore-utils.yaml b/mail/openeuler.org/mailcore-utils.yaml index e4434115..f4af862e 100644 --- a/mail/openeuler.org/mailcore-utils.yaml +++ b/mail/openeuler.org/mailcore-utils.yaml @@ -58,7 +58,7 @@ spec: value: restpass # default domain name that will be created when starts up - name: DEFAULT_DOMAIN_NAME - value: openeuler + value: openeuler.org # default mail lists that will created when starts up, if not specified, container will scan all of the list file in templates folder. - name: DEFAULT_MAIL_LISTS value: community,announce,council,dev,infra,marketing,user-committee -- Gitee From 37fbf072982e7d9eb5b23dab01b944b6c08b6a34 Mon Sep 17 00:00:00 2001 From: TommyLike Date: Thu, 12 Sep 2019 16:32:55 +0800 Subject: [PATCH 3/3] Typo fix --- mail/dockerfile/core_utils/mailman-core-utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mail/dockerfile/core_utils/mailman-core-utils.py b/mail/dockerfile/core_utils/mailman-core-utils.py index a64a2dbe..94aec30c 100644 --- a/mail/dockerfile/core_utils/mailman-core-utils.py +++ b/mail/dockerfile/core_utils/mailman-core-utils.py @@ -154,7 +154,7 @@ def running_templates_server(): # exit with 0 when sigterm signal received. signal.signal(signal.SIGTERM, httpd_signal_handler) httpd.serve_forever() - except (IndentationError, SignalException): + except (InterruptedError, SignalException): pass print("template server ends") httpd.server_close() -- Gitee