Restart running monitors if no heartbeat (#3952)

This commit is contained in:
Louis Lam
2023-11-01 09:36:12 +08:00
committed by GitHub
parent 9f170a68d7
commit c43223a16d
2 changed files with 106 additions and 1 deletions

View File

@@ -3,7 +3,7 @@ const dayjs = require("dayjs");
const axios = require("axios");
const { Prometheus } = require("../prometheus");
const { log, UP, DOWN, PENDING, MAINTENANCE, flipStatus, TimeLogger, MAX_INTERVAL_SECOND, MIN_INTERVAL_SECOND,
SQL_DATETIME_FORMAT
SQL_DATETIME_FORMAT, isDev, sleep, getRandomInt
} = require("../../src/util");
const { tcping, ping, dnsResolve, checkCertificate, checkStatusCode, getTotalClientInRoom, setting, mssqlQuery, postgresQuery, mysqlQuery, mqttAsync, setSetting, httpNtlm, radius, grpcQuery,
redisPingAsync, mongodbPing, kafkaProducerAsync, getOidcTokenClientCredentials, rootCertificatesFingerprints
@@ -328,6 +328,16 @@ class Monitor extends BeanModel {
}
}
// Evil
if (isDev) {
if (process.env.EVIL_RANDOM_MONITOR_SLEEP === "SURE") {
if (getRandomInt(0, 100) === 0) {
log.debug("evil", `[${this.name}] Evil mode: Random sleep: ` + beatInterval * 10000);
await sleep(beatInterval * 10000);
}
}
}
// Expose here for prometheus update
// undefined if not https
let tlsInfo = undefined;
@@ -995,6 +1005,7 @@ class Monitor extends BeanModel {
if (! this.isStop) {
log.debug("monitor", `[${this.name}] SetTimeout for next check.`);
this.heartbeatInterval = setTimeout(safeBeat, beatInterval * 1000);
this.lastScheduleBeatTime = dayjs();
} else {
log.info("monitor", `[${this.name}] isStop = true, no next check.`);
}
@@ -1004,7 +1015,9 @@ class Monitor extends BeanModel {
/** Get a heartbeat and handle errors */
const safeBeat = async () => {
try {
this.lastStartBeatTime = dayjs();
await beat();
this.lastEndBeatTime = dayjs();
} catch (e) {
console.trace(e);
UptimeKumaServer.errorLog(e, false);
@@ -1013,6 +1026,9 @@ class Monitor extends BeanModel {
if (! this.isStop) {
log.info("monitor", "Try to restart the monitor");
this.heartbeatInterval = setTimeout(safeBeat, this.interval * 1000);
this.lastScheduleBeatTime = dayjs();
} else {
log.info("monitor", "isStop = true, no next check.");
}
}
};