Merge branch 'feature/add_prometheus_metrics'

2025-10-31 03:19:20 +08:00 · 2021-07-27 23:21:37 +08:00
parent 0f0a6299c0 36436ed4ef
commit cc6f1d7487
4 changed files with 216 additions and 2 deletions
--- a/package-lock.json
+++ b/package-lock.json
@@ -68,16 +68,57 @@
      "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.9.2.tgz",
      "integrity": "sha512-VZMYa7+fXHdwIq1TDhSXoVmSPEGM/aa+6Aiq3nVVJ9bXr24zScr+NlKFKC3iPljA7ho/GAZr+d2jOf5GIRC30Q=="
    },
+    "@types/accepts": {
+      "version": "1.3.5",
+      "resolved": "https://registry.npmjs.org/@types/accepts/-/accepts-1.3.5.tgz",
+      "integrity": "sha512-jOdnI/3qTpHABjM5cx1Hc0sKsPoYCp+DP/GJRGtDlPd7fiV9oXGGIcjW/ZOxLIvjGz8MA+uMZI9metHlgqbgwQ==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
+    "@types/body-parser": {
+      "version": "1.19.1",
+      "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.1.tgz",
+      "integrity": "sha512-a6bTJ21vFOGIkwM0kzh9Yr89ziVxq4vYH2fQ6N8AeipEzai/cFK6aGMArIkUeIdRIgpwQa+2bXiLuUJCpSf2Cg==",
+      "requires": {
+        "@types/connect": "*",
+        "@types/node": "*"
+      }
+    },
    "@types/component-emitter": {
      "version": "1.2.10",
      "resolved": "https://registry.npmjs.org/@types/component-emitter/-/component-emitter-1.2.10.tgz",
      "integrity": "sha512-bsjleuRKWmGqajMerkzox19aGbscQX5rmmvvXl3wlIp5gMG1HgkiwPxsN5p070fBDKTNSPgojVbuY1+HWMbFhg=="
    },
+    "@types/connect": {
+      "version": "3.4.35",
+      "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz",
+      "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
+    "@types/content-disposition": {
+      "version": "0.5.4",
+      "resolved": "https://registry.npmjs.org/@types/content-disposition/-/content-disposition-0.5.4.tgz",
+      "integrity": "sha512-0mPF08jn9zYI0n0Q/Pnz7C4kThdSt+6LD4amsrYDDpgBfrVWa3TcCOxKX1zkGgYniGagRv8heN2cbh+CAn+uuQ=="
+    },
    "@types/cookie": {
      "version": "0.4.1",
      "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.4.1.tgz",
      "integrity": "sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q=="
    },
+    "@types/cookies": {
+      "version": "0.7.7",
+      "resolved": "https://registry.npmjs.org/@types/cookies/-/cookies-0.7.7.tgz",
+      "integrity": "sha512-h7BcvPUogWbKCzBR2lY4oqaZbO3jXZksexYJVFvkrFeLgbZjQkU4x8pRq6eg2MHXQhY0McQdqmmsxRWlVAHooA==",
+      "requires": {
+        "@types/connect": "*",
+        "@types/express": "*",
+        "@types/keygrip": "*",
+        "@types/node": "*"
+      }
+    },
    "@types/cors": {
      "version": "2.8.12",
      "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz",
@@ -89,11 +130,94 @@
      "integrity": "sha512-LfZwXoGUDo0C3me81HXgkBg5CTQYb6xzEl+fNmbO4JdRiSKQ8A0GD1OBBvKAIsbCUgoyAty7m99GqqMQe784ew==",
      "dev": true
    },
+    "@types/express": {
+      "version": "4.17.13",
+      "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz",
+      "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==",
+      "requires": {
+        "@types/body-parser": "*",
+        "@types/express-serve-static-core": "^4.17.18",
+        "@types/qs": "*",
+        "@types/serve-static": "*"
+      }
+    },
+    "@types/express-serve-static-core": {
+      "version": "4.17.24",
+      "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.24.tgz",
+      "integrity": "sha512-3UJuW+Qxhzwjq3xhwXm2onQcFHn76frIYVbTu+kn24LFxI+dEhdfISDFovPB8VpEgW8oQCTpRuCe+0zJxB7NEA==",
+      "requires": {
+        "@types/node": "*",
+        "@types/qs": "*",
+        "@types/range-parser": "*"
+      }
+    },
+    "@types/http-assert": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/@types/http-assert/-/http-assert-1.5.1.tgz",
+      "integrity": "sha512-PGAK759pxyfXE78NbKxyfRcWYA/KwW17X290cNev/qAsn9eQIxkH4shoNBafH37wewhDG/0p1cHPbK6+SzZjWQ=="
+    },
+    "@types/http-errors": {
+      "version": "1.8.1",
+      "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-1.8.1.tgz",
+      "integrity": "sha512-e+2rjEwK6KDaNOm5Aa9wNGgyS9oSZU/4pfSMMPYNOfjvFI0WVXm29+ITRFr6aKDvvKo7uU1jV68MW4ScsfDi7Q=="
+    },
+    "@types/keygrip": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@types/keygrip/-/keygrip-1.0.2.tgz",
+      "integrity": "sha512-GJhpTepz2udxGexqos8wgaBx4I/zWIDPh/KOGEwAqtuGDkOUJu5eFvwmdBX4AmB8Odsr+9pHCQqiAqDL/yKMKw=="
+    },
+    "@types/koa": {
+      "version": "2.13.4",
+      "resolved": "https://registry.npmjs.org/@types/koa/-/koa-2.13.4.tgz",
+      "integrity": "sha512-dfHYMfU+z/vKtQB7NUrthdAEiSvnLebvBjwHtfFmpZmB7em2N3WVQdHgnFq+xvyVgxW5jKDmjWfLD3lw4g4uTw==",
+      "requires": {
+        "@types/accepts": "*",
+        "@types/content-disposition": "*",
+        "@types/cookies": "*",
+        "@types/http-assert": "*",
+        "@types/http-errors": "*",
+        "@types/keygrip": "*",
+        "@types/koa-compose": "*",
+        "@types/node": "*"
+      }
+    },
+    "@types/koa-compose": {
+      "version": "3.2.5",
+      "resolved": "https://registry.npmjs.org/@types/koa-compose/-/koa-compose-3.2.5.tgz",
+      "integrity": "sha512-B8nG/OoE1ORZqCkBVsup/AKcvjdgoHnfi4pZMn5UwAPCbhk/96xyv284eBYW8JlQbQ7zDmnpFr68I/40mFoIBQ==",
+      "requires": {
+        "@types/koa": "*"
+      }
+    },
+    "@types/mime": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz",
+      "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw=="
+    },
    "@types/node": {
      "version": "16.3.3",
      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.3.3.tgz",
      "integrity": "sha512-8h7k1YgQKxKXWckzFCMfsIwn0Y61UK6tlD6y2lOb3hTOIMlK3t9/QwHOhc81TwU+RMf0As5fj7NPjroERCnejQ=="
    },
+    "@types/qs": {
+      "version": "6.9.7",
+      "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz",
+      "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw=="
+    },
+    "@types/range-parser": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz",
+      "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw=="
+    },
+    "@types/serve-static": {
+      "version": "1.13.10",
+      "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz",
+      "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==",
+      "requires": {
+        "@types/mime": "^1",
+        "@types/node": "*"
+      }
+    },
    "@vitejs/plugin-legacy": {
      "version": "1.4.4",
      "resolved": "https://registry.npmjs.org/@vitejs/plugin-legacy/-/plugin-legacy-1.4.4.tgz",
@@ -591,6 +715,11 @@
      "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==",
      "dev": true
    },
+    "bintrees": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz",
+      "integrity": "sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ="
+    },
    "block-stream": {
      "version": "0.0.9",
      "resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.9.tgz",
@@ -2038,6 +2167,11 @@
      "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=",
      "dev": true
    },
+    "lodash.get": {
+      "version": "4.4.2",
+      "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
+      "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk="
+    },
    "lodash.includes": {
      "version": "4.3.0",
      "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
@@ -2791,6 +2925,11 @@
      "integrity": "sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==",
      "dev": true
    },
+    "pkginfo": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/pkginfo/-/pkginfo-0.4.1.tgz",
+      "integrity": "sha1-tUGO8EOd5UJfxJlQQtztFPsqhP8="
+    },
    "posix-character-classes": {
      "version": "0.1.1",
      "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz",
@@ -2879,6 +3018,37 @@
      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
    },
+    "prom-client": {
+      "version": "13.1.0",
+      "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-13.1.0.tgz",
+      "integrity": "sha512-jT9VccZCWrJWXdyEtQddCDszYsiuWj5T0ekrPszi/WEegj3IZy6Mm09iOOVM86A4IKMWq8hZkT2dD9MaSe+sng==",
+      "requires": {
+        "tdigest": "^0.1.1"
+      }
+    },
+    "prometheus-api-metrics": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/prometheus-api-metrics/-/prometheus-api-metrics-3.2.0.tgz",
+      "integrity": "sha512-JekPhtIBLGX8HxD2EndvBsLU6ZQ1JVVqyHWVfm5CposUOqgBHXnUVFW6x5Ux2gykpdej/5LLM3dU9V8Ma7GfkA==",
+      "requires": {
+        "@types/express": "^4.17.8",
+        "@types/express-serve-static-core": "^4.17.12",
+        "@types/koa": "^2.11.4",
+        "debug": "^3.2.6",
+        "lodash.get": "^4.4.2",
+        "pkginfo": "^0.4.1"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.2.7",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
+          "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
+          "requires": {
+            "ms": "^2.1.1"
+          }
+        }
+      }
+    },
    "proxy-addr": {
      "version": "2.0.7",
      "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
@@ -3563,6 +3733,14 @@
      "resolved": "https://registry.npmjs.org/tcp-ping/-/tcp-ping-0.1.1.tgz",
      "integrity": "sha1-At1/QrW/fXy3jVt6rO+hVf2PfAw="
    },
+    "tdigest": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz",
+      "integrity": "sha1-Ljyyw56kSeVdHmzZEReszKRYgCE=",
+      "requires": {
+        "bintrees": "1.0.1"
+      }
+    },
    "tildify": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz",
--- a/package.json
+++ b/package.json
@@ -37,6 +37,8 @@
        "jsonwebtoken": "^8.5.1",
        "nodemailer": "^6.6.3",
        "password-hash": "^1.2.2",
+        "prom-client": "^13.1.0",
+        "prometheus-api-metrics": "^3.2.0",
        "redbean-node": "0.0.20",
        "socket.io": "^4.1.3",
        "socket.io-client": "^4.1.3",
--- a/server/model/monitor.js
+++ b/server/model/monitor.js
@@ -1,4 +1,4 @@
-
+const Prometheus = require('prom-client');
 const https = require('https');
 const dayjs = require("dayjs");
 const utc = require('dayjs/plugin/utc')
@@ -18,6 +18,26 @@ const customAgent = new https.Agent({
    maxCachedSessions: 0
 });

+const commonLabels = [
+    'monitor_name',
+    'monitor_type',
+    'monitor_url',
+    'monitor_hostname',
+    'monitor_port',
+]
+
+const monitor_response_time = new Prometheus.Gauge({
+    name: 'monitor_response_time',
+    help: 'Monitor Response Time (ms)',
+    labelNames: commonLabels
+});
+
+const monitor_status = new Prometheus.Gauge({
+    name: 'monitor_status',
+    help: 'Monitor Status (1 = UP, 0= DOWN)',
+    labelNames: commonLabels
+});
+
 /**
 * status:
 *      0 = DOWN
@@ -56,6 +76,14 @@ class Monitor extends BeanModel {
        let previousBeat = null;
        let retries = 0;

+        const monitorLabelValues = {
+                monitor_name: this.name,
+                monitor_type: this.type,
+                monitor_url: this.url,
+                monitor_hostname: this.hostname,
+                monitor_port: this.port
+        }
+
        const beat = async () => {

            if (! previousBeat) {
@@ -191,6 +219,8 @@ class Monitor extends BeanModel {
                bean.important = false;
            }

+            monitor_status.set(monitorLabelValues, bean.status)
+
            if (bean.status === UP) {
                console.info(`Monitor #${this.id} '${this.name}': Successful Response: ${bean.ping} ms | Interval: ${this.interval} seconds | Type: ${this.type}`)
            } else if (bean.status === PENDING) {
@@ -199,6 +229,8 @@ class Monitor extends BeanModel {
                console.warn(`Monitor #${this.id} '${this.name}': Failing: ${bean.msg} | Type: ${this.type}`)
            }

+            monitor_response_time.set(monitorLabelValues, bean.ping)
+
            io.to(this.user_id).emit("heartbeat", bean.toJSON());

            await R.store(bean)
--- a/server/server.js
+++ b/server/server.js
@@ -15,7 +15,7 @@ const gracefulShutdown = require('http-graceful-shutdown');
 const Database = require("./database");
 const {sleep} = require("./util");
 const args = require('args-parser')(process.argv);
-
+const apiMetrics = require('prometheus-api-metrics');
 const version = require('../package.json').version;
 const hostname = args.host || "0.0.0.0"
 const port = args.port || 3001
@@ -57,6 +57,8 @@ let needSetup = false;

    console.log("Adding route")
    app.use('/', express.static("dist"));
+    app.use(apiMetrics())
+

    app.get('*', function(request, response, next) {
        response.sendFile(process.cwd() + '/dist/index.html');