[Dovecot] Simplify Docker image

[Dovecot] Set Dovecot plugins dynamically via file and exclude Solr if not enabled [Dovecot] Add new quarantine notification script
2019-01-29 00:11:12 +01:00
parent aab692301e
commit f493d3a957
5 changed files with 177 additions and 57 deletions
--- a/data/Dockerfiles/dovecot/Dockerfile
+++ b/data/Dockerfiles/dovecot/Dockerfile
@@ -64,6 +64,9 @@ RUN apt-get update && apt-get -y --no-install-recommends install \
  libregexp-common-perl \
  liburi-perl \
  lzma-dev \
+  python-redis \
+  python-jinja2 \
+  python-mysql.connector \
  make \
  mysql-client \
  procps \
@@ -73,9 +76,8 @@ RUN apt-get update && apt-get -y --no-install-recommends install \
  syslog-ng \
  syslog-ng-core \
  syslog-ng-mod-redis \
-  && rm -rf /var/lib/apt/lists/*
-
-RUN curl https://www.dovecot.org/releases/2.3/dovecot-$DOVECOT_VERSION.tar.gz | tar xvz  \
+  && rm -rf /var/lib/apt/lists/* \
+  && curl https://www.dovecot.org/releases/2.3/dovecot-$DOVECOT_VERSION.tar.gz | tar xvz  \
  && cd dovecot-$DOVECOT_VERSION \
  && ./configure --with-solr --with-mysql --with-ldap --with-lzma --with-lz4 --with-ssl=openssl --with-notify=inotify --with-storages=mdbox,sdbox,maildir,mbox,imapc,pop3c --with-bzlib --with-zlib --enable-hardening \
  && make -j3 \
@@ -89,15 +91,19 @@ RUN curl https://www.dovecot.org/releases/2.3/dovecot-$DOVECOT_VERSION.tar.gz |
  && make install \
  && make clean \
  && cd .. \
-  && rm -rf dovecot-2.3-pigeonhole-$PIGEONHOLE_VERSION
+  && rm -rf dovecot-2.3-pigeonhole-$PIGEONHOLE_VERSION \
+  && cpanm Data::Uniqid Mail::IMAPClient String::Util \
+  && groupadd -g 5000 vmail \
+  && groupadd -g 401 dovecot \
+  && groupadd -g 402 dovenull \
+  && useradd -g vmail -u 5000 vmail -d /var/vmail \
+  && useradd -c "Dovecot unprivileged user" -d /dev/null -u 401 -g dovecot -s /bin/false dovecot \
+  && useradd -c "Dovecot login user" -d /dev/null -u 402 -g dovenull -s /bin/false dovenull \
+  && touch /etc/default/locale \
+  && apt-get purge -y build-essential automake autotools-dev default-libmysqlclient-dev libbz2-dev libcurl4-openssl-dev libexpat1-dev liblz-dev liblz4-dev liblzma-dev libpam-dev libssl-dev lzma-dev \
+  && apt-get autoremove --purge -y \
+  && rm -rf /tmp/* /var/tmp/*

-RUN cpanm Data::Uniqid Mail::IMAPClient String::Util
-RUN echo '* * * * *   root  /usr/local/bin/imapsync_cron.pl 2>&1 | /usr/bin/logger' > /etc/cron.d/imapsync
-RUN echo '30 3 * * *  vmail /usr/local/bin/doveadm quota recalc -A' > /etc/cron.d/dovecot-sync
-RUN echo '* * * * *   vmail /usr/local/bin/trim_logs.sh >> /dev/console 2>&1' > /etc/cron.d/trim_logs
-RUN echo '25 * * * *  vmail /usr/local/bin/maildir_gc.sh >> /dev/console 2>&1' > /etc/cron.d/maildir_gc
-RUN echo '30 1 * * *  root  /usr/local/bin/sa-rules.sh  >> /dev/console 2>&1' > /etc/cron.d/sa-rules
-RUN echo '0 2 * * *   root  /usr/bin/curl http://solr:8983/solr/dovecot/update?optimize=true >> /dev/console 2>&1' > /etc/cron.d/solr-optimize 
 COPY trim_logs.sh /usr/local/bin/trim_logs.sh
 COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
 COPY imapsync /usr/local/bin/imapsync
@@ -112,32 +118,7 @@ COPY maildir_gc.sh /usr/local/bin/maildir_gc.sh
 COPY docker-entrypoint.sh /
 COPY supervisord.conf /etc/supervisor/supervisord.conf
 COPY stop-supervisor.sh /usr/local/sbin/stop-supervisor.sh
-
-RUN chmod +x /usr/local/lib/dovecot/sieve/rspamd-pipe-ham \
-  /usr/local/lib/dovecot/sieve/rspamd-pipe-spam \
-  /usr/local/bin/imapsync_cron.pl \
-  /usr/local/bin/postlogin.sh \
-  /usr/local/bin/imapsync \
-  /usr/local/bin/trim_logs.sh \
-  /usr/local/bin/sa-rules.sh \
-  /usr/local/bin/maildir_gc.sh \
-  /usr/local/sbin/stop-supervisor.sh
-
-RUN groupadd -g 5000 vmail \
-  && groupadd -g 401 dovecot \
-  && groupadd -g 402 dovenull \
-  && useradd -g vmail -u 5000 vmail -d /var/vmail \
-  && useradd -c "Dovecot unprivileged user" -d /dev/null -u 401 -g dovecot -s /bin/false dovecot \
-  && useradd -c "Dovecot login user" -d /dev/null -u 402 -g dovenull -s /bin/false dovenull
-
-RUN touch /etc/default/locale
-RUN apt-get purge -y build-essential automake autotools-dev default-libmysqlclient-dev libbz2-dev libcurl4-openssl-dev libexpat1-dev liblz-dev liblz4-dev liblzma-dev libpam-dev libssl-dev lzma-dev \
-  && apt-get autoremove --purge -y
+COPY quarantine_notify.py /usr/local/bin/quarantine_notify.py

 ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
-
-RUN rm -rf \
-  /tmp/* \
-  /var/tmp/*
-
--- a/data/Dockerfiles/dovecot/docker-entrypoint.sh
+++ b/data/Dockerfiles/dovecot/docker-entrypoint.sh
@@ -7,11 +7,16 @@ while ! mysqladmin status --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${D
  sleep 2
 done

-# Hard-code env vars to scripts due to cron not passing them to the perl script
-sed -i "/^\$DBUSER/c\\\$DBUSER='${DBUSER}';" /usr/local/bin/imapsync_cron.pl
-sed -i "/^\$DBPASS/c\\\$DBPASS='${DBPASS}';" /usr/local/bin/imapsync_cron.pl
-sed -i "/^\$DBNAME/c\\\$DBNAME='${DBNAME}';" /usr/local/bin/imapsync_cron.pl
-sed -i "s/LOG_LINES/${LOG_LINES}/g" /usr/local/bin/trim_logs.sh
+# Hard-code env vars to scripts due to cron not passing them to the scripts
+sed -i "s/__DBUSER__/${DBUSER}/g" /usr/local/bin/imapsync_cron.pl
+sed -i "s/__DBPASS__/${DBPASS}/g" /usr/local/bin/imapsync_cron.pl
+sed -i "s/__DBNAME__/${DBNAME}/g" /usr/local/bin/imapsync_cron.pl
+
+sed -i "s/__DBUSER__/${DBUSER}/g" /usr/local/bin/quarantine_notify.py
+sed -i "s/__DBPASS__/${DBPASS}/g" /usr/local/bin/quarantine_notify.py
+sed -i "s/__DBNAME__/${DBNAME}/g" /usr/local/bin/quarantine_notify.py
+
+sed -i "s/__LOG_LINES__/${LOG_LINES}/g" /usr/local/bin/trim_logs.sh

 # Create missing directories
 [[ ! -d /usr/local/etc/dovecot/sql/ ]] && mkdir -p /usr/local/etc/dovecot/sql/
@@ -87,7 +92,17 @@ EOF

 echo -n ${ACL_ANYONE} > /usr/local/etc/dovecot/acl_anyone

-# Create userdb dict for Dovecot
+if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+echo -n 'quota acl zlib listescape mail_crypt mail_crypt_acl mail_log notify' > /usr/local/etc/dovecot/mail_plugins
+echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve listescape mail_crypt mail_crypt_acl notify mail_log' > /usr/local/etc/dovecot/mail_plugins_imap
+echo -n 'quota sieve acl zlib listescape mail_crypt mail_crypt_acl' > /usr/local/etc/dovecot/mail_plugins_lmtp
+else
+echo -n 'quota acl zlib listescape mail_crypt mail_crypt_acl mail_log notify fts fts_solr' > /usr/local/etc/dovecot/mail_plugins
+echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve listescape mail_crypt mail_crypt_acl notify mail_log fts fts_solr' > /usr/local/etc/dovecot/mail_plugins_imap
+echo -n 'quota sieve acl zlib listescape mail_crypt mail_crypt_acl fts fts_solr' > /usr/local/etc/dovecot/mail_plugins_lmtp
+fi
+chmod 644 /usr/local/etc/dovecot/mail_plugins /usr/local/etc/dovecot/mail_plugins_imap /usr/local/etc/dovecot/mail_plugins_lmtp /templates/quarantine.tpl
+
 cat <<EOF > /usr/local/etc/dovecot/sql/dovecot-dict-sql-userdb.conf
 driver = mysql
 connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
@@ -142,6 +157,24 @@ chown -R vmail:vmail /var/vmail/sieve
 chown -R vmail:vmail /var/volatile
 adduser vmail tty
 chmod g+rw /dev/console
+chmod +x /usr/local/lib/dovecot/sieve/rspamd-pipe-ham \
+  /usr/local/lib/dovecot/sieve/rspamd-pipe-spam \
+  /usr/local/bin/imapsync_cron.pl \
+  /usr/local/bin/postlogin.sh \
+  /usr/local/bin/imapsync \
+  /usr/local/bin/trim_logs.sh \
+  /usr/local/bin/sa-rules.sh \
+  /usr/local/bin/maildir_gc.sh \
+  /usr/local/sbin/stop-supervisor.sh
+
+# Setup cronjobs
+echo '* * * * *    root  /usr/local/bin/imapsync_cron.pl 2>&1 | /usr/bin/logger' > /etc/cron.d/imapsync
+echo '30 3 * * *   vmail /usr/local/bin/doveadm quota recalc -A' > /etc/cron.d/dovecot-sync
+echo '* * * * *    vmail /usr/local/bin/trim_logs.sh >> /dev/console 2>&1' > /etc/cron.d/trim_logs
+echo '25 * * * *   vmail /usr/local/bin/maildir_gc.sh >> /dev/console 2>&1' > /etc/cron.d/maildir_gc
+echo '30 1 * * *   root  /usr/local/bin/sa-rules.sh  >> /dev/console 2>&1' > /etc/cron.d/sa-rules
+echo '0 2 * * *    root  /usr/bin/curl http://solr:8983/solr/dovecot/update?optimize=true >> /dev/console 2>&1' > /etc/cron.d/solr-optimize
+echo '*/20 * * * * vmail /usr/local/bin/quarantine_notify.py >> /dev/console 2>&1' > /etc/cron.d/quarantine_notify

 # Fix more than 1 hardlink issue
 touch /etc/crontab /etc/cron.*/*
--- a/data/Dockerfiles/dovecot/imapsync_cron.pl
+++ b/data/Dockerfiles/dovecot/imapsync_cron.pl
@@ -26,16 +26,12 @@ sub qqw($) {
  return @values
 }

-$DBNAME = '';
-$DBUSER = '';
-$DBPASS = '';
-
 $run_dir="/tmp";
-$dsn = "DBI:mysql:database=" . $DBNAME . ";mysql_socket=/var/run/mysqld/mysqld.sock";
+$dsn = 'DBI:mysql:database=__DBNAME__;mysql_socket=/var/run/mysqld/mysqld.sock';
 $lock_file = $run_dir . "/imapsync_busy";
 $lockmgr = LockFile::Simple->make(-autoclean => 1, -max => 1);
 $lockmgr->lock($lock_file) || die "can't lock ${lock_file}";
-$dbh = DBI->connect($dsn, $DBUSER, $DBPASS, {
+$dbh = DBI->connect($dsn, '__DBUSER__', '__DBPASS__', {
  mysql_auto_reconnect => 1,
  mysql_enable_utf8mb4 => 1
 });
--- a/data/Dockerfiles/dovecot/quarantine_notify.py
+++ b/data/Dockerfiles/dovecot/quarantine_notify.py
@@ -0,0 +1,110 @@
+#!/usr/bin/python
+
+import smtplib
+import os
+import mysql.connector
+from email.MIMEMultipart import MIMEMultipart
+from email.MIMEText import MIMEText
+from email.Utils import COMMASPACE, formatdate
+import cgi
+import jinja2
+from jinja2 import Template
+import json
+import redis
+import time
+
+while True:
+  try:
+    r = redis.StrictRedis(host='redis', decode_responses=True, port=6379, db=0)
+    r.ping()
+  except Exception as ex:
+    print '%s - trying again...'  % (ex)
+    time.sleep(3)
+  else:
+    break
+
+time_now = int(time.time())
+
+def query_mysql(query, headers = True, update = False):
+  while True:
+    try:
+      cnx = mysql.connector.connect(unix_socket = '/var/run/mysqld/mysqld.sock', user='__DBUSER__', passwd='__DBPASS__', database='__DBNAME__', charset="utf8")
+    except Exception as ex:
+      print '%s - trying again...'  % (ex)
+      time.sleep(3)
+    else:
+      break
+  cur = cnx.cursor()
+  cur.execute(query)
+  if not update:
+    result = []
+    columns = tuple( [d[0].decode('utf8') for d in cur.description] )
+    for row in cur:
+      if headers: 
+        result.append(dict(zip(columns, row)))
+      else:
+        result.append(row)
+    cur.close()
+    cnx.close()
+    return result
+  else:
+    cnx.commit()
+    cur.close()
+    cnx.close()
+
+def notify_rcpt(rcpt, msg_count):
+  meta_query = query_mysql('SELECT id, subject, sender, created FROM quarantine WHERE notified = 0 AND rcpt = "%s"' % (rcpt))
+  if r.get('Q_HTML'):
+    try:
+      template = Template(r.get('Q_HTML'))
+    except:
+      print "Error: Cannot parse quarantine template, falling back to default template."
+    with open('/templates/quarantine.tpl') as file_:
+      template = Template(file_.read())
+  else:
+    with open('/templates/quarantine.tpl') as file_:
+      template = Template(file_.read())
+  html = template.render(meta=meta_query, counter=msg_count)
+  count = 0
+  while count < 15:
+    try:
+      server = smtplib.SMTP('postfix', 589, 'quarntine')
+      server.ehlo()
+      msg = MIMEMultipart('alternative')
+      msg['From'] = r.get('Q_SENDER') or "quarantine@localhost"
+      msg['Subject'] = r.get('Q_SUBJ') or "Spam Quarantine Notification"
+      msg['Date'] = formatdate(localtime = True)
+      text = "You have %d new items" % (msg_count)
+      text_part = MIMEText(text, 'plain')
+      html_part = MIMEText(html, 'html')
+      msg.attach(text_part)
+      msg.attach(html_part)
+      msg['To'] = str(rcpt)
+      text = msg.as_string()
+      server.sendmail(msg['From'], msg['To'], text)
+      server.quit()
+      for res in meta_query:
+        query_mysql('UPDATE quarantine SET notified = 1 WHERE id = "%d"' % (res['id']), update = True)
+      r.hset('Q_LAST_NOTIFIED', record['rcpt'], time_now)
+      break
+    except Exception as ex:
+      print '%s'  % (ex)
+      time.sleep(3)
+
+records = query_mysql('SELECT count(id) AS counter, rcpt FROM quarantine WHERE notified = 0 GROUP BY rcpt')
+
+for record in records:
+  last_notification = int(r.hget('Q_LAST_NOTIFIED', record['rcpt'])) or 0
+  attrs_json = query_mysql('SELECT attributes FROM mailbox WHERE username = "%s"' % (record['rcpt']))
+  attrs = json.loads(str(attrs_json[0]['attributes']))
+  if attrs['quarantine_notification'] == 'hourly':
+    if last_notification == 0 or (last_notification + 3600) > time_now:
+      notify_rcpt(record['rcpt'], record['counter'])
+  elif attrs['quarantine_notification'] == 'daily':
+    if last_notification == 0 or (last_notification + 86400) > time_now:
+      notify_rcpt(record['rcpt'], record['counter'])
+  elif attrs['quarantine_notification'] == 'weekly':
+    if last_notification == 0 or (last_notification + 604800) > time_now:
+      notify_rcpt(record['rcpt'], record['counter'])
+  else:
+    break
--- a/data/Dockerfiles/dovecot/trim_logs.sh
+++ b/data/Dockerfiles/dovecot/trim_logs.sh
@@ -7,12 +7,12 @@ catch_non_zero() {
    echo "Command ${CMD} failed to execute, exit code was ${EC}"
  fi
 }
-catch_non_zero "/usr/bin/redis-cli -h redis LTRIM ACME_LOG 0 LOG_LINES"
-catch_non_zero "/usr/bin/redis-cli -h redis LTRIM POSTFIX_MAILLOG 0 LOG_LINES"
-catch_non_zero "/usr/bin/redis-cli -h redis LTRIM DOVECOT_MAILLOG 0 LOG_LINES"
-catch_non_zero "/usr/bin/redis-cli -h redis LTRIM SOGO_LOG 0 LOG_LINES"
-catch_non_zero "/usr/bin/redis-cli -h redis LTRIM NETFILTER_LOG 0 LOG_LINES"
-catch_non_zero "/usr/bin/redis-cli -h redis LTRIM AUTODISCOVER_LOG 0 LOG_LINES"
-catch_non_zero "/usr/bin/redis-cli -h redis LTRIM API_LOG 0 LOG_LINES"
-catch_non_zero "/usr/bin/redis-cli -h redis LTRIM RL_LOG 0 LOG_LINES"
+catch_non_zero "/usr/bin/redis-cli -h redis LTRIM ACME_LOG 0 __LOG_LINES__"
+catch_non_zero "/usr/bin/redis-cli -h redis LTRIM POSTFIX_MAILLOG 0 __LOG_LINES__"
+catch_non_zero "/usr/bin/redis-cli -h redis LTRIM DOVECOT_MAILLOG 0 __LOG_LINES__"
+catch_non_zero "/usr/bin/redis-cli -h redis LTRIM SOGO_LOG 0 __LOG_LINES__"
+catch_non_zero "/usr/bin/redis-cli -h redis LTRIM NETFILTER_LOG 0 __LOG_LINES__"
+catch_non_zero "/usr/bin/redis-cli -h redis LTRIM AUTODISCOVER_LOG 0 __LOG_LINES__"
+catch_non_zero "/usr/bin/redis-cli -h redis LTRIM API_LOG 0 __LOG_LINES__"
+catch_non_zero "/usr/bin/redis-cli -h redis LTRIM RL_LOG 0 __LOG_LINES__"