Merge pull request #4827 from mailcow/staging

2022-10a

.github/ISSUE_TEMPLATE/pr_to_nighty_template.yml

@@ -0,0 +1,13 @@
+## :memo:  Brief description
+<!-- Diff summary - START -->
+<!-- Diff summary - END -->
+
+
+## :computer:  Commits
+<!-- Diff commits - START -->
+<!-- Diff commits - END -->
+
+
+## :file_folder:  Modified files
+<!-- Diff files - START -->
+<!-- Diff files - END -->

.github/workflows/close_old_issues_and_prs.yml

@@ -14,7 +14,7 @@ jobs:
       pull-requests: write
     steps:
       - name: Mark/Close Stale Issues and Pull Requests 🗑️
-        uses: actions/stale@v6.0.0
+        uses: actions/stale@v6.0.1
         with:
           repo-token: ${{ secrets.STALE_ACTION_PAT }}
           days-before-stale: 60

.github/workflows/pr_to_nightly.yml

@@ -0,0 +1,25 @@
+name: Create PR to merge to nightly from staging
+on:
+  push:
+    branches:
+      - staging
+jobs:
+  action-pull-request:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Run the Action
+        uses: devops-infra/action-pull-request@v0.5.1
+        with:
+          github_token: ${{ secrets.PRTONIGHTLY_ACTION_PAT }}
+          title: Automatic PR to nightly from ${{ github.event.repository.updated_at}}
+          assignee: DerLinkman
+          source_branch: staging
+          target_branch: nightly
+          reviewer: DerLinkman
+          label: upstream
+          template: .github/ISSUE_TEMPLATE/pr_to_nighty_template.yml
+          get_diff: true

data/Dockerfiles/backup/Dockerfile

@@ -0,0 +1,3 @@
+FROM debian:bullseye-slim
+
+RUN apt update && apt install pigz

data/Dockerfiles/netfilter/server.py

@@ -346,6 +346,8 @@ def snat4(snat_target):
     rule.dst = '!' + rule.src
     target = rule.create_target("SNAT")
     target.to_source = snat_target
+    match = rule.create_match("comment")
+    match.comment = f'{int(round(time.time()))}'
     return rule
 
   while not quit_now:
@@ -356,16 +358,23 @@ def snat4(snat_target):
         table.refresh()
         chain = iptc.Chain(table, 'POSTROUTING')
         table.autocommit = False
-        if get_snat4_rule() not in chain.rules:
-          logCrit('Added POSTROUTING rule for source network %s to SNAT target %s' % (get_snat4_rule().src, snat_target))
-          chain.insert_rule(get_snat4_rule())
-          table.commit()
-        else:
-          for position, item in enumerate(chain.rules):
-            if item == get_snat4_rule():
-              if position != 0:
-                chain.delete_rule(get_snat4_rule())
-          table.commit()
+        new_rule = get_snat4_rule()
+        for position, rule in enumerate(chain.rules):
+          match = all((
+            new_rule.get_src() == rule.get_src(),
+            new_rule.get_dst() == rule.get_dst(),
+            new_rule.target.parameters == rule.target.parameters,
+            new_rule.target.name == rule.target.name
+          ))
+          if position == 0:
+            if not match:
+              logInfo(f'Added POSTROUTING rule for source network {new_rule.src} to SNAT target {snat_target}')
+              chain.insert_rule(new_rule)
+          else:
+            if match:
+              logInfo(f'Remove rule for source network {new_rule.src} to SNAT target {snat_target} from POSTROUTING chain at position {position}')
+              chain.delete_rule(rule)
+        table.commit()
         table.autocommit = True
       except:
         print('Error running SNAT4, retrying...')

data/web/inc/ajax/qitem_details.php

@@ -127,6 +127,7 @@ elseif (!empty($_GET['id']) && ctype_alnum($_GET['id'])) {
     $data['fuzzy_hashes'] = json_decode($mailc['fuzzy_hashes']);
     // Get text/plain content
     $data['text_plain'] = $mail_parser->getMessageBody('text');
+    if (!json_encode($data['text_plain'])) $data['text_plain'] = '';
     // Get html content and convert to text
     $data['text_html'] = $html2text->convert($mail_parser->getMessageBody('html'));
     if (empty($data['text_plain']) && empty($data['text_html'])) {

data/web/inc/prerequisites.inc.php

@@ -195,9 +195,65 @@ require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/sessions.inc.php';
 // Set language
 if (!isset($_SESSION['mailcow_locale']) && !isset($_COOKIE['mailcow_locale'])) {
   if ($DETECT_LANGUAGE && isset($_SERVER['HTTP_ACCEPT_LANGUAGE'])) {
-    $header_lang = strtolower(substr($_SERVER['HTTP_ACCEPT_LANGUAGE'], 0, 2));
-    if (array_key_exists($header_lang, $AVAILABLE_LANGUAGES)) {
-      $_SESSION['mailcow_locale'] = $header_lang;
+    // regex inspired from @GabrielAnderson on http://stackoverflow.com/questions/6038236/http-accept-language
+    preg_match_all('/([a-z]{1,8}(-[a-z]{1,8})*)\s*(;\s*q\s*=\s*(1|0\.[0-9]+))?/i', $_SERVER['HTTP_ACCEPT_LANGUAGE'], $lang_parse);
+
+    $langs = $lang_parse[1];
+    $ranks = $lang_parse[4];
+
+    // (create an associative array 'language' => 'preference')
+    $lang2pref = array();
+    for ($i=0; $i<count($langs); $i++) {
+      $lang2pref[strtolower($langs[$i])] = (float) (!empty($ranks[$i]) ? $ranks[$i] : 1);
+    }
+
+    // (comparison function for uksort)
+    $cmpLangs = function ($a, $b) use ($lang2pref) {
+      if ($lang2pref[$a] > $lang2pref[$b])
+        return -1;
+      elseif ($lang2pref[$a] < $lang2pref[$b])
+        return 1;
+      elseif (strlen($a) > strlen($b))
+        return -1;
+      elseif (strlen($a) < strlen($b))
+        return 1;
+      else
+        return 0;
+    };
+
+    // sort the languages by prefered language and by the most specific region
+    uksort($lang2pref, $cmpLangs);
+
+    // generate language array without the region part
+    $AVAILABLE_BASE_LANGUAGES=array();
+    foreach ($AVAILABLE_LANGUAGES as $code => $lang) {
+      $base_code = substr($code, 0, 2);
+      if (!array_key_exists($base_code, $AVAILABLE_BASE_LANGUAGES)) {
+        $AVAILABLE_BASE_LANGUAGES[$base_code] = $code;
+      }
+    }
+
+    // Find a perfect match or partial match
+    // Match en-gb or en
+    foreach ($lang2pref as $lang => $q) {
+      if (array_key_exists($lang, $AVAILABLE_LANGUAGES)) {
+        $_SESSION['mailcow_locale'] = $lang;
+        break;
+      } elseif (array_key_exists($lang, $AVAILABLE_BASE_LANGUAGES)) {
+        $_SESSION['mailcow_locale'] = $AVAILABLE_BASE_LANGUAGES[$lang];
+        break;
+      }
+    }
+
+    // Try suggest match
+    // e.g. suggest en-gb when only en-us is provided
+    if (!isset($_COOKIE['mailcow_locale'])) {
+      foreach ($lang2pref as $lang => $q) {
+        if (array_key_exists(substr($lang, 0, 2), $AVAILABLE_BASE_LANGUAGES)) {
+          $_SESSION['mailcow_locale'] = $AVAILABLE_BASE_LANGUAGES[substr($lang, 0, 2)];
+          break;
+        }
+      }
     }
   }
   else {
@@ -215,7 +271,7 @@ if (isset($_GET['lang']) && array_key_exists($_GET['lang'], $AVAILABLE_LANGUAGES
 /*
  * load language
  */
-$lang = json_decode(file_get_contents($_SERVER['DOCUMENT_ROOT'] . '/lang/lang.en.json'), true);
+$lang = json_decode(file_get_contents($_SERVER['DOCUMENT_ROOT'] . '/lang/lang.en-gb.json'), true);
 
 $langFile = $_SERVER['DOCUMENT_ROOT'] . '/lang/lang.'.$_SESSION['mailcow_locale'].'.json';
 if(file_exists($langFile)) {

data/web/inc/vars.inc.php

@@ -76,33 +76,35 @@ $autodiscover_config = array(
 $DETECT_LANGUAGE = true;
 
 // Change default language
-$DEFAULT_LANG = 'en';
+$DEFAULT_LANG = 'en-gb';
 
 // Available languages
 // https://www.iso.org/obp/ui/#search
-// https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes
+// https://en.wikipedia.org/wiki/IETF_language_tag
 $AVAILABLE_LANGUAGES = array(
-  'cs' => 'Čeština (Czech)',
-  'da' => 'Danish (Dansk)',
-  'de' => 'Deutsch (German)',
-  'en' => 'English',
-  'es' => 'Español (Spanish)',
-  'fi' => 'Suomi (Finish)',
-  'fr' => 'Français (French)',
-  'hu' => 'Magyar (Hungarian)',
-  'it' => 'Italiano (Italian)',
-  'ko' => '한국어 (Korean)',
-  'lv' => 'latviešu (Latvian)',
-  'nl' => 'Nederlands (Dutch)',
-  'pl' => 'Język Polski (Polish)',
-  'pt' => 'Português (Portuguese)',
-  'ro' => 'Română (Romanian)',
-  'ru' => 'Pусский (Russian)',
-  'sk' => 'Slovenčina (Slovak)',
-  'sv' => 'Svenska (Swedish)',
-  'tr' => 'Türkçe (Turkish)',
-  'uk' => 'Українська (Ukrainian)',
-  'zh' => '中文 (Chinese)'
+  // 'ca-es' => 'Català (Catalan)',
+  'cs-cz' => 'Čeština (Czech)',
+  'da-dk' => 'Danish (Dansk)',
+  'de-de' => 'Deutsch (German)',
+  'en-gb' => 'English',
+  'es-es' => 'Español (Spanish)',
+  'fi-fi' => 'Suomi (Finish)',
+  'fr-fr' => 'Français (French)',
+  'hu-hu' => 'Magyar (Hungarian)',
+  'it-it' => 'Italiano (Italian)',
+  'ko-kr' => '한국어 (Korean)',
+  'lv-lv' => 'latviešu (Latvian)',
+  'nl-nl' => 'Nederlands (Dutch)',
+  'pl-pl' => 'Język Polski (Polish)',
+  'pt-pt' => 'Português (Portuguese)',
+  'ro-ro' => 'Română (Romanian)',
+  'ru-ru' => 'Pусский (Russian)',
+  'sk-sk' => 'Slovenčina (Slovak)',
+  'sv-se' => 'Svenska (Swedish)',
+  'tr-tr' => 'Türkçe (Turkish)',
+  'uk-ua' => 'Українська (Ukrainian)',
+  'zh-cn' => '简体中文 (Simplified Chinese)',
+  'zh-tw' => '繁體中文 (Traditional Chinese)',
 );
 
 // Change theme (default: lumen)

data/web/templates/base.twig

@@ -32,12 +32,12 @@
       <ul class="nav navbar-nav navbar-right">
         {% if mailcow_locale %}
         <li class="dropdown{% if available_languages|length == 1 %}lang-link-disabled{% endif %}">
-          <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false"><span class="flag-icon flag-icon-{{ mailcow_locale }}"></span><span class="caret"></span></a>
+          <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false"><span class="flag-icon flag-icon-{{ mailcow_locale[-2:] }}"></span><span class="caret"></span></a>
           <ul class="dropdown-menu" role="menu">
             {% for key, val in available_languages %}
             <li{% if mailcow_locale == key %} class="active"{% endif %}>
               <a href="?{{ query_string({'lang': key}) }}">
-                <span class="flag-icon flag-icon-{{ key }}"></span>{{ val }}
+                <span class="flag-icon flag-icon-{{ key[-2:] }}"></span>{{ val }}
               </a>
             </li>
             {% endfor %}
@@ -172,7 +172,7 @@ function recursiveBase64StrToArrayBuffer(obj) {
     // TFA, CSRF, Alerts in footer.inc.php
     // Other general functions in mailcow.js
     {% for alert_type, alert_msg in alerts %}
-    mailcow_alert_box('{{ alert_msg|raw }}', '{{ alert_type }}');
+    mailcow_alert_box('{{ alert_msg|raw|e("js") }}', '{{ alert_type }}');
     {% endfor %}
 
     // Confirm TFA modal

data/web/templates/index.twig

@@ -45,13 +45,13 @@
             </div>
             {% if not oauth2_request %}
             <button type="button" {% if available_languages|length == 1 %}disabled="true"{% endif %} class="btn btn-xs-lg btn-default pull-right dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
-            <span class="flag-icon flag-icon-{{ mailcow_locale }}"></span> <span class="caret"></span>
+            <span class="flag-icon flag-icon-{{ mailcow_locale[-2:] }}"></span> <span class="caret"></span>
             </button>
             <ul class="dropdown-menu pull-right login">
               {% for key, val in available_languages %}
                 <li{% if mailcow_locale == key %} class="active"{% endif %}>
                   <a href="?{{ query_string({'lang': key}) }}">
-                    <span class="flag-icon flag-icon-{{ key }}"></span>{{ val }}
+                    <span class="flag-icon flag-icon-{{ key[-2:] }}"></span>{{ val }}
                   </a>
                 </li>
               {% endfor %}

docker-compose.yml

@@ -41,7 +41,7 @@ services:
             - mysql
 
     redis-mailcow:
-      image: redis:6-alpine
+      image: redis:7-alpine
       volumes:
         - redis-vol-1:/data/
       restart: always
@@ -424,7 +424,7 @@ services:
             - acme
 
     netfilter-mailcow:
-      image: mailcow/netfilter:1.48
+      image: mailcow/netfilter:1.49
       stop_grace_period: 30s
       depends_on:
         - dovecot-mailcow

helper-scripts/add-new-lang-keys.php

@@ -26,7 +26,7 @@ if(empty($targetLang)) {
 }
 
 // load master lang
-$masterLang = file_get_contents(__DIR__.'/../data/web/lang/lang.en.json');
+$masterLang = file_get_contents(__DIR__.'/../data/web/lang/lang.en-gb.json');
 $masterLang = json_decode($masterLang, true);
 
 // load target lang

helper-scripts/backup_and_restore.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-DEBIAN_DOCKER_IMAGE="debian:bullseye-slim"
+DEBIAN_DOCKER_IMAGE="mailcow/backup:1.0"
 
 if [[ ! -z ${MAILCOW_BACKUP_LOCATION} ]]; then
   BACKUP_LOCATION="${MAILCOW_BACKUP_LOCATION}"
@@ -52,6 +52,15 @@ BACKUP_LOCATION=$(echo ${BACKUP_LOCATION} | sed 's#/$##')
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 COMPOSE_FILE=${SCRIPT_DIR}/../docker-compose.yml
 ENV_FILE=${SCRIPT_DIR}/../.env
+THREADS=$(echo ${THREADS:-1})
+
+if ! [[ "${THREADS}" =~ ^[1-9]+$ ]] ; then
+  echo "Thread input is not a number!"
+  exit 1
+elif [[ "${THREADS}" =~ ^[1-9]+$ ]] ; then
+  echo "Using ${THREADS} Thread(s) for this run." 
+  echo "Notice: You can set the Thread count with the THREADS Variable before you run this script."
+fi
 
 if [ ! -f ${COMPOSE_FILE} ]; then
   echo "Compose file not found"
@@ -99,32 +108,32 @@ function backup() {
       docker run --name mailcow-backup --rm \
         -v ${BACKUP_LOCATION}/mailcow-${DATE}:/backup:z \
         -v $(docker volume ls -qf name=^${CMPS_PRJ}_vmail-vol-1$):/vmail:ro,z \
-        ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="gzip --rsyncable" -Pcvpf /backup/backup_vmail.tar.gz /vmail
+        ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="pigz --rsyncable -p ${THREADS}" -Pcvpf /backup/backup_vmail.tar.gz /vmail
       ;;&
     crypt|all)
       docker run --name mailcow-backup --rm \
         -v ${BACKUP_LOCATION}/mailcow-${DATE}:/backup:z \
         -v $(docker volume ls -qf name=^${CMPS_PRJ}_crypt-vol-1$):/crypt:ro,z \
-        ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="gzip --rsyncable" -Pcvpf /backup/backup_crypt.tar.gz /crypt
+        ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="pigz --rsyncable -p ${THREADS}" -Pcvpf /backup/backup_crypt.tar.gz /crypt
       ;;&
     redis|all)
       docker exec $(docker ps -qf name=redis-mailcow) redis-cli save
       docker run --name mailcow-backup --rm \
         -v ${BACKUP_LOCATION}/mailcow-${DATE}:/backup:z \
         -v $(docker volume ls -qf name=^${CMPS_PRJ}_redis-vol-1$):/redis:ro,z \
-        ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="gzip --rsyncable" -Pcvpf /backup/backup_redis.tar.gz /redis
+        ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="pigz --rsyncable -p ${THREADS}" -Pcvpf /backup/backup_redis.tar.gz /redis
       ;;&
     rspamd|all)
       docker run --name mailcow-backup --rm \
         -v ${BACKUP_LOCATION}/mailcow-${DATE}:/backup:z \
         -v $(docker volume ls -qf name=^${CMPS_PRJ}_rspamd-vol-1$):/rspamd:ro,z \
-        ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="gzip --rsyncable" -Pcvpf /backup/backup_rspamd.tar.gz /rspamd
+        ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="pigz --rsyncable -p ${THREADS}" -Pcvpf /backup/backup_rspamd.tar.gz /rspamd
       ;;&
     postfix|all)
       docker run --name mailcow-backup --rm \
         -v ${BACKUP_LOCATION}/mailcow-${DATE}:/backup:z \
         -v $(docker volume ls -qf name=^${CMPS_PRJ}_postfix-vol-1$):/postfix:ro,z \
-        ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="gzip --rsyncable" -Pcvpf /backup/backup_postfix.tar.gz /postfix
+        ${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="pigz --rsyncable -p ${THREADS}" -Pcvpf /backup/backup_postfix.tar.gz /postfix
       ;;&
     mysql|all)
       SQLIMAGE=$(grep -iEo '(mysql|mariadb)\:.+' ${COMPOSE_FILE})
@@ -191,7 +200,7 @@ function restore() {
       docker run -it --name mailcow-backup --rm \
         -v ${RESTORE_LOCATION}:/backup:z \
         -v $(docker volume ls -qf name=^${CMPS_PRJ}_vmail-vol-1$):/vmail:z \
-        ${DEBIAN_DOCKER_IMAGE} /bin/tar -Pxvzf /backup/backup_vmail.tar.gz
+        ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_vmail.tar.gz
       docker start $(docker ps -aqf name=dovecot-mailcow)
       echo
       echo "In most cases it is not required to run a full resync, you can run the command printed below at any time after testing wether the restore process broke a mailbox:"
@@ -210,7 +219,7 @@ function restore() {
       docker run -it --name mailcow-backup --rm \
         -v ${RESTORE_LOCATION}:/backup:z \
         -v $(docker volume ls -qf name=^${CMPS_PRJ}_redis-vol-1$):/redis:z \
-        ${DEBIAN_DOCKER_IMAGE} /bin/tar -Pxvzf /backup/backup_redis.tar.gz
+        ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_redis.tar.gz
       docker start $(docker ps -aqf name=redis-mailcow)
       ;;
     crypt)
@@ -218,7 +227,7 @@ function restore() {
       docker run -it --name mailcow-backup --rm \
         -v ${RESTORE_LOCATION}:/backup:z \
         -v $(docker volume ls -qf name=^${CMPS_PRJ}_crypt-vol-1$):/crypt:z \
-        ${DEBIAN_DOCKER_IMAGE} /bin/tar -Pxvzf /backup/backup_crypt.tar.gz
+        ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_crypt.tar.gz
       docker start $(docker ps -aqf name=dovecot-mailcow)
       ;;
     rspamd)
@@ -226,7 +235,7 @@ function restore() {
       docker run -it --name mailcow-backup --rm \
         -v ${RESTORE_LOCATION}:/backup:z \
         -v $(docker volume ls -qf name=^${CMPS_PRJ}_rspamd-vol-1$):/rspamd:z \
-        ${DEBIAN_DOCKER_IMAGE} /bin/tar -Pxvzf /backup/backup_rspamd.tar.gz
+        ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_rspamd.tar.gz
       docker start $(docker ps -aqf name=rspamd-mailcow)
       ;;
     postfix)
@@ -234,7 +243,7 @@ function restore() {
       docker run -it --name mailcow-backup --rm \
         -v ${RESTORE_LOCATION}:/backup:z \
         -v $(docker volume ls -qf name=^${CMPS_PRJ}_postfix-vol-1$):/postfix:z \
-        ${DEBIAN_DOCKER_IMAGE} /bin/tar -Pxvzf /backup/backup_postfix.tar.gz
+        ${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_postfix.tar.gz
       docker start $(docker ps -aqf name=postfix-mailcow)
       ;;
     mysql|mariadb)

helper-scripts/check_translations.rb

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-MASTER="en"
+MASTER="en-gb"
 
 DIR = "#{__dir__}/.."