[Docker API] Use TLS encryption for communication with "on-the-fly" created key paris (non-exposed)

[Docker API] Create pipe to pass Rspamd UI worker password [Dovecot] Pull Spamassassin ruleset to be read by Rspamd (MANY THANKS to Peer Heinlein!) [Dovecot] Garbage collector for deleted maildirs (set keep time via MAILDIR_GC_TIME which defaults to 1440 minutes) [Web] Flush memcached after mailbox item changes, fixes #1808 [Web] Fix duplicate IDs, fixes #1792 [Compose] Use SQL sockets [PHP-FPM] Update APCu and Redis libs [Dovecot] Encrypt maildir with global key pair in crypt-vol-1 (BACKUP!), also fixes #1791 [Web] Fix deletion of spam aliases [Helper] Add "crypt" to backup script [Helper] Override file for external SQL socket (not supported!) [Compose] New images for Rspamd, PHP-FPM, SOGo, Dovecot, Docker API, Watchdog, ACME, Postfix
2018-09-29 22:01:23 +02:00
parent 96c985abad
commit 0fb43f4916
49 changed files with 11437 additions and 419 deletions
--- a/data/Dockerfiles/dovecot/Dockerfile
+++ b/data/Dockerfiles/dovecot/Dockerfile
@@ -14,6 +14,7 @@ RUN apt-get update && apt-get -y --no-install-recommends install \
  cpanminus \
  curl \
  default-libmysqlclient-dev \
+  dnsutils \
  libjson-webtoken-perl \
  libcgi-pm-perl \
  libcrypt-openssl-rsa-perl \
@@ -88,10 +89,11 @@ RUN curl https://www.dovecot.org/releases/2.3/dovecot-$DOVECOT_VERSION.tar.gz |
  && rm -rf dovecot-2.3-pigeonhole-$PIGEONHOLE_VERSION

 RUN cpanm Data::Uniqid Mail::IMAPClient String::Util
-RUN echo '* * * * *   root   /usr/local/bin/imapsync_cron.pl' > /etc/cron.d/imapsync
-RUN echo '30 3 * * *   vmail  /usr/local/bin/doveadm quota recalc -A' > /etc/cron.d/dovecot-sync
-RUN echo '* * * * *   root  /usr/local/bin/trim_logs.sh >> /dev/stdout 2>&1' > /etc/cron.d/trim_logs
-
+RUN echo '* * * * *   root  /usr/local/bin/imapsync_cron.pl' > /etc/cron.d/imapsync
+RUN echo '30 3 * * *  vmail /usr/local/bin/doveadm quota recalc -A' > /etc/cron.d/dovecot-sync
+RUN echo '* * * * *   vmail /usr/local/bin/trim_logs.sh >> /dev/stdout 2>&1' > /etc/cron.d/trim_logs
+RUN echo '30 2 * * *  vmail /usr/local/bin/maildir_gc.sh >> /dev/stdout 2>&1' > /etc/cron.d/maildir_gc
+RUN echo '30 1 * * *  root  /usr/local/bin/sa-rules.sh  >> /dev/stdout 2>&1' > /etc/cron.d/sa-rules
 COPY trim_logs.sh /usr/local/bin/trim_logs.sh
 COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
 COPY imapsync /usr/local/bin/imapsync
@@ -101,6 +103,8 @@ COPY report-spam.sieve /usr/local/lib/dovecot/sieve/report-spam.sieve
 COPY report-ham.sieve /usr/local/lib/dovecot/sieve/report-ham.sieve
 COPY rspamd-pipe-ham /usr/local/lib/dovecot/sieve/rspamd-pipe-ham
 COPY rspamd-pipe-spam /usr/local/lib/dovecot/sieve/rspamd-pipe-spam
+COPY sa-rules.sh /usr/local/bin/sa-rules.sh
+COPY maildir_gc.sh /usr/local/bin/maildir_gc.sh
 COPY docker-entrypoint.sh /
 COPY supervisord.conf /etc/supervisor/supervisord.conf

@@ -109,7 +113,9 @@ RUN chmod +x /usr/local/lib/dovecot/sieve/rspamd-pipe-ham \
  /usr/local/bin/imapsync_cron.pl \
  /usr/local/bin/postlogin.sh \
  /usr/local/bin/imapsync \
-  /usr/local/bin/trim_logs.sh
+  /usr/local/bin/trim_logs.sh \
+  /usr/local/bin/sa-rules.sh \
+  /usr/local/bin/maildir_gc.sh

 RUN groupadd -g 5000 vmail \
  && groupadd -g 401 dovecot \
--- a/data/Dockerfiles/dovecot/docker-entrypoint.sh
+++ b/data/Dockerfiles/dovecot/docker-entrypoint.sh
@@ -2,7 +2,7 @@
 set -e

 # Wait for MySQL to warm-up
-while ! mysqladmin ping --host mysql -u${DBUSER} -p${DBPASS} --silent; do
+while ! mysqladmin ping --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} --silent; do
  echo "Waiting for database to come up..."
  sleep 2
 done
@@ -15,6 +15,7 @@ sed -i "s/LOG_LINES/${LOG_LINES}/g" /usr/local/bin/trim_logs.sh

 # Create missing directories
 [[ ! -d /usr/local/etc/dovecot/sql/ ]] && mkdir -p /usr/local/etc/dovecot/sql/
+[[ ! -d /var/vmail/_garbage ]] && mkdir -p /var/vmail/_garbage
 [[ ! -d /var/vmail/sieve ]] && mkdir -p /var/vmail/sieve
 [[ ! -d /etc/sogo ]] && mkdir -p /etc/sogo

@@ -23,7 +24,7 @@ DBPASS=$(echo ${DBPASS} | sed 's/"/\\"/g')

 # Create quota dict for Dovecot
 cat <<EOF > /usr/local/etc/dovecot/sql/dovecot-dict-sql-quota.conf
-connect = "host=mysql dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
+connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
 map {
  pattern = priv/quota/storage
  table = quota2
@@ -40,7 +41,7 @@ EOF

 # Create dict used for sieve pre and postfilters
 cat <<EOF > /usr/local/etc/dovecot/sql/dovecot-dict-sql-sieve_before.conf
-connect = "host=mysql dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
+connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
 map {
  pattern = priv/sieve/name/\$script_name
  table = sieve_before
@@ -62,7 +63,7 @@ map {
 EOF

 cat <<EOF > /usr/local/etc/dovecot/sql/dovecot-dict-sql-sieve_after.conf
-connect = "host=mysql dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
+connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
 map {
  pattern = priv/sieve/name/\$script_name
  table = sieve_after
@@ -87,7 +88,7 @@ EOF
 # Create userdb dict for Dovecot
 cat <<EOF > /usr/local/etc/dovecot/sql/dovecot-dict-sql-userdb.conf
 driver = mysql
-connect = "host=mysql dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
+connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
 user_query = SELECT CONCAT('maildir:/var/vmail/',maildir) AS mail, 5000 AS uid, 5000 AS gid, concat('*:bytes=', quota) AS quota_rule FROM mailbox WHERE username = '%u' AND active = '1'
 iterate_query = SELECT username FROM mailbox WHERE active='1';
 EOF
@@ -95,7 +96,7 @@ EOF
 # Create pass dict for Dovecot
 cat <<EOF > /usr/local/etc/dovecot/sql/dovecot-dict-sql-passdb.conf
 driver = mysql
-connect = "host=mysql dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
+connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS} ssl_verify_server_cert=no ssl_ca=/etc/ssl/certs/ca-certificates.crt"
 default_pass_scheme = SSHA256
 password_query = SELECT password FROM mailbox WHERE username = '%u' AND domain IN (SELECT domain FROM domain WHERE domain='%d' AND active='1') AND JSON_EXTRACT(attributes, '$.force_pw_update') NOT LIKE '%%1%%'
 EOF
@@ -106,6 +107,7 @@ cat /usr/local/etc/dovecot/sieve_after > /var/vmail/sieve/global.sieve
 # Check permissions of vmail directory.
 # Do not do this every start-up, it may take a very long time. So we use a stat check here.
 if [[ $(stat -c %U /var/vmail/) != "vmail" ]] ; then chown -R vmail:vmail /var/vmail ; fi
+if [[ $(stat -c %U /var/vmail/_garbage) != "vmail" ]] ; then chown -R vmail:vmail /var/vmail/_garbage ; fi

 # Create random master for SOGo sieve features
 RAND_USER=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 16 | head -n 1)
@@ -139,7 +141,7 @@ touch /etc/crontab /etc/cron.*/*

 # Clean stopped imapsync jobs
 rm -f /tmp/imapsync_busy.lock
-IMAPSYNC_TABLE=$(mysql -h mysql-mailcow -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SHOW TABLES LIKE 'imapsync'" -Bs)
-[[ ! -z ${IMAPSYNC_TABLE} ]] && mysql -h mysql-mailcow -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "UPDATE imapsync SET is_running='0'"
+IMAPSYNC_TABLE=$(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SHOW TABLES LIKE 'imapsync'" -Bs)
+[[ ! -z ${IMAPSYNC_TABLE} ]] && mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "UPDATE imapsync SET is_running='0'"

 exec "$@"
--- a/data/Dockerfiles/dovecot/maildir_gc.sh
+++ b/data/Dockerfiles/dovecot/maildir_gc.sh
@@ -0,0 +1,2 @@
+#/bin/bash
+[ -d /var/vmail/_garbage/ ] && /usr/bin/find /var/vmail/_garbage/ -mindepth 1 -maxdepth 1 -type d -cmin +${MAILDIR_GC_TIME} -exec rm -r {} \;
--- a/data/Dockerfiles/dovecot/postlogin.sh
+++ b/data/Dockerfiles/dovecot/postlogin.sh
@@ -1,4 +1,3 @@
 #!/bin/sh
-
 export MASTER_USER=$USER
 exec "$@"
--- a/data/Dockerfiles/dovecot/sa-rules.sh
+++ b/data/Dockerfiles/dovecot/sa-rules.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+[[ ! -d /tmp/sa-rules-heinlein ]] && mkdir -p /tmp/sa-rules-heinlein
+if [[ ! -f /etc/rspamd/custom/sa-rules-heinlein ]]; then
+  HASH_SA_RULES=0
+else
+  HASH_SA_RULES=$(cat /etc/rspamd/custom/sa-rules-heinlein | md5sum | cut -d' ' -f1)
+fi
+
+curl http://www.spamassassin.heinlein-support.de/$(dig txt 1.4.3.spamassassin.heinlein-support.de +short | tr -d '"').tar.gz --output /tmp/sa-rules.tar.gz
+if [[ -f /tmp/sa-rules.tar.gz ]]; then
+  tar xfvz /tmp/sa-rules.tar.gz -C /tmp/sa-rules-heinlein
+  # create complete list of rules in a single file
+  cat /tmp/sa-rules-heinlein/*cf > /etc/rspamd/custom/sa-rules-heinlein
+  # Only restart rspamd-mailcow when rules changed
+  if [[ $(cat /etc/rspamd/custom/sa-rules-heinlein | md5sum | cut -d' ' -f1) != ${HASH_SA_RULES} ]]; then
+    CONTAINER_NAME=rspamd-mailcow
+    CONTAINER_ID=$(curl --silent --insecure https://dockerapi/containers/json | \
+      jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], id: .Id}" | \
+      jq -rc "select( .name | tostring | contains(\"${CONTAINER_NAME}\")) | .id")
+    if [[ ! -z ${CONTAINER_ID} ]]; then
+      curl --silent --insecure -XPOST https://dockerapi/containers/${CONTAINER_ID}/restart
+    fi
+  fi
+fi
+rm -r /tmp/sa-rules-heinlein /tmp/sa-rules.tar.gz
--- a/data/Dockerfiles/dovecot/trim_logs.sh
+++ b/data/Dockerfiles/dovecot/trim_logs.sh
@@ -1,8 +1,7 @@
 #!/bin/bash
-
-redis-cli -h redis LTRIM ACME_LOG 0 LOG_LINES
-redis-cli -h redis LTRIM POSTFIX_MAILLOG 0 LOG_LINES
-redis-cli -h redis LTRIM DOVECOT_MAILLOG 0 LOG_LINES
-redis-cli -h redis LTRIM SOGO_LOG 0 LOG_LINES
-redis-cli -h redis LTRIM NETFILTER_LOG 0 LOG_LINES
-redis-cli -h redis LTRIM AUTODISCOVER_LOG 0 LOG_LINES
+/usr/bin/redis-cli -h redis LTRIM ACME_LOG 0 LOG_LINES
+/usr/bin/redis-cli -h redis LTRIM POSTFIX_MAILLOG 0 LOG_LINES
+/usr/bin/redis-cli -h redis LTRIM DOVECOT_MAILLOG 0 LOG_LINES
+/usr/bin/redis-cli -h redis LTRIM SOGO_LOG 0 LOG_LINES
+/usr/bin/redis-cli -h redis LTRIM NETFILTER_LOG 0 LOG_LINES
+/usr/bin/redis-cli -h redis LTRIM AUTODISCOVER_LOG 0 LOG_LINES