[Compose] New images, Nginx checks for SOGo before bootstrapping

[PHP-FPM] Some more modules (primarily for Horde) [Fail2ban] Do not log matches of local and private ips [Watchdog] Some changes in log system for further processing (wip) [ACME] Fixes #745
2017-11-14 10:44:00 +01:00
parent c2d9928f8f
commit 84a7a1a2e7
5 changed files with 45 additions and 44 deletions
--- a/data/Dockerfiles/watchdog/watchdog.sh
+++ b/data/Dockerfiles/watchdog/watchdog.sh
@@ -28,13 +28,19 @@ progress() {
  [[ ${CURRENT} -gt ${TOTAL} ]] && return
  [[ ${CURRENT} -lt 0 ]] && CURRENT=0
  PERCENT=$(( 200 * ${CURRENT} / ${TOTAL} % 2 + 100 * ${CURRENT} / ${TOTAL} ))
-  echo -ne "$(date) - ${SERVICE} health level: \e[7m${PERCENT}%\e[0m (${CURRENT}/${TOTAL}), health trend: "
-  [[ ${DIFF} =~ ^-[1-9] ]] && echo -en '[\e[41m  \e[0m] ' || echo -en '[\e[42m  \e[0m] '
-  echo "(${DIFF})"
+  log_msg "${SERVICE} health level: ${PERCENT}% (${CURRENT}/${TOTAL}), health trend: ${DIFF}"
+  log_data "$(printf "%d,%d,%d,%d" ${PERCENT} ${CURRENT} ${TOTAL} ${DIFF})" "${SERVICE}"
 }

-log_to_redis() {
-	redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}")\"}"
+log_msg() {
+	redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}")\"}" > /dev/null
+  echo $(date) $(printf '%s\n' "${1}")
+}
+
+log_data() {
+  [[ -z ${1} ]] && return 1
+  [[ -z ${2} ]] && return 2
+	redis-cli -h redis LPUSH WATCHDOG_DATA "{\"time\":\"$(date +%s)\",\"service\":\"data\",\"$(printf '%s' "${2}")\":\"$(printf '%s' "${1}")\"}" > /dev/null
 }

 function mail_error() {
@@ -43,8 +49,7 @@ function mail_error() {
  RCPT_DOMAIN=$(echo ${1} | awk -F @ {'print $NF'})
  RCPT_MX=$(dig +short ${RCPT_DOMAIN} mx | sort -n | awk '{print $2; exit}')
  if [[ -z ${RCPT_MX} ]]; then
-    log_to_redis "Cannot determine MX for ${1}, skipping email notification..."
-    echo "Cannot determine MX for ${1}"
+    log_msg "Cannot determine MX for ${1}, skipping email notification..."
    return 1
  fi
  ./smtp-cli --missing-modules-ok \
@@ -54,6 +59,7 @@ function mail_error() {
    --from="watchdog@${MAILCOW_HOSTNAME}" \
    --server="${RCPT_MX}" \
    --hello-host=${MAILCOW_HOSTNAME}
+  log_msg "Sent notification email to ${1}"
 }


@@ -66,8 +72,8 @@ get_container_ip() {
    sleep 1
    CONTAINER_ID=$(curl --silent http://dockerapi:8080/containers/json | jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], id: .Id}" | jq -rc "select( .name | tostring | contains(\"${1}\")) | .id")
    if [[ ! -z ${CONTAINER_ID} ]]; then
-    	CONTAINER_IP=$(curl --silent http://dockerapi:8080/containers/${CONTAINER_ID}/json | jq -r '.NetworkSettings.Networks[].IPAddress')
-	fi
+      CONTAINER_IP=$(curl --silent http://dockerapi:8080/containers/${CONTAINER_ID}/json | jq -r '.NetworkSettings.Networks[].IPAddress')
+    fi
    LOOP_C=$((LOOP_C + 1))
  done
  [[ ${LOOP_C} -gt 5 ]] && echo 240.0.0.0 || echo ${CONTAINER_IP}
@@ -253,9 +259,8 @@ dns_checks() {
 (
 while true; do
  if ! nginx_checks; then
-    log_to_redis "Nginx hit error limit"
+    log_msg "Nginx hit error limit"
    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "nginx-mailcow"
-    echo -e "\e[31m$(date) - Nginx hit error limit\e[0m"
    echo nginx-mailcow > /tmp/com_pipe
  fi
 done
@@ -265,9 +270,8 @@ BACKGROUND_TASKS+=($!)
 (
 while true; do
  if ! mysql_checks; then
-    log_to_redis "MySQL hit error limit"
+    log_msg "MySQL hit error limit"
    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "mysql-mailcow"
-    echo -e "\e[31m$(date) - MySQL hit error limit\e[0m"
    echo mysql-mailcow > /tmp/com_pipe
  fi
 done
@@ -277,9 +281,8 @@ BACKGROUND_TASKS+=($!)
 (
 while true; do
  if ! phpfpm_checks; then
-    log_to_redis "PHP-FPM hit error limit"
+    log_msg "PHP-FPM hit error limit"
    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "php-fpm-mailcow"
-    echo -e "\e[31m$(date) - PHP-FPM hit error limit\e[0m"
    echo php-fpm-mailcow > /tmp/com_pipe
  fi
 done
@@ -289,9 +292,8 @@ BACKGROUND_TASKS+=($!)
 (
 while true; do
  if ! sogo_checks; then
-    log_to_redis "SOGo hit error limit"
+    log_msg "SOGo hit error limit"
    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "sogo-mailcow"
-    echo -e "\e[31m$(date) - SOGo hit error limit\e[0m"
    echo sogo-mailcow > /tmp/com_pipe
  fi
 done
@@ -301,9 +303,8 @@ BACKGROUND_TASKS+=($!)
 (
 while true; do
  if ! postfix_checks; then
-    log_to_redis "Postfix hit error limit"
+    log_msg "Postfix hit error limit"
    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "postfix-mailcow"
-    echo -e "\e[31m$(date) - Postfix hit error limit\e[0m"
    echo postfix-mailcow > /tmp/com_pipe
  fi
 done
@@ -313,9 +314,8 @@ BACKGROUND_TASKS+=($!)
 (
 while true; do
  if ! dovecot_checks; then
-    log_to_redis "Dovecot hit error limit"
+    log_msg "Dovecot hit error limit"
    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "dovecot-mailcow"
-    echo -e "\e[31m$(date) - Dovecot hit error limit\e[0m"
    echo dovecot-mailcow > /tmp/com_pipe
  fi
 done
@@ -325,9 +325,8 @@ BACKGROUND_TASKS+=($!)
 (
 while true; do
  if ! dns_checks; then
-    log_to_redis "Unbound hit error limit"
+    log_msg "Unbound hit error limit"
    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "unbound-mailcow"
-    echo -e "\e[31m$(date) - Unbound hit error limit\e[0m"
    #echo unbound-mailcow > /tmp/com_pipe
  fi
 done
@@ -337,9 +336,8 @@ BACKGROUND_TASKS+=($!)
 (
 while true; do
  if ! rspamd_checks; then
-    log_to_redis "Rspamd hit error limit"
+    log_msg "Rspamd hit error limit"
    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "rspamd-mailcow"
-    echo -e "\e[31m$(date) - Rspamd hit error limit\e[0m"
    echo rspamd-mailcow > /tmp/com_pipe
  fi
 done
@@ -351,8 +349,7 @@ BACKGROUND_TASKS+=($!)
 while true; do
  for bg_task in ${BACKGROUND_TASKS[*]}; do
    if ! kill -0 ${bg_task} 1>&2; then
-      echo "Worker ${bg_task} died, stopping watchdog and waiting for respawn..."
-      log_to_redis "Worker ${bg_task} died, stopping watchdog and waiting for respawn..."
+      log_msg "Worker ${bg_task} died, stopping watchdog and waiting for respawn..."
      kill -TERM 1
    fi
    sleep 10
@@ -366,7 +363,7 @@ while true; do
  while nc -z dockerapi 8080; do
    sleep 3
  done
-  echo "Cannot find dockerapi-mailcow, waiting to recover..."
+  log_msg "Cannot find dockerapi-mailcow, waiting to recover..."
  kill -STOP ${BACKGROUND_TASKS[*]}
  until nc -z dockerapi 8080; do
    sleep 3
@@ -385,11 +382,10 @@ while true; do
    sleep 3
    CONTAINER_ID=$(curl --silent http://dockerapi:8080/containers/json | jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], id: .Id}" | jq -rc "select( .name | tostring | contains(\"${com_pipe_answer}\")) | .id")
    if [[ ! -z ${CONTAINER_ID} ]]; then
-      log_to_redis "Sending restart command to ${CONTAINER_ID}..."
-      echo "Sending restart command to ${CONTAINER_ID}..."
+      log_msg "Sending restart command to ${CONTAINER_ID}..."
      curl --silent -XPOST http://dockerapi:8080/containers/${CONTAINER_ID}/restart
    fi
-    echo "Wait for restarted container to settle and continue watching..."
+    log_msg "Wait for restarted container to settle and continue watching..."
    sleep 30s
    kill -CONT ${BACKGROUND_TASKS[*]}
    kill -USR1 ${BACKGROUND_TASKS[*]}