A complete maintenance planning system has been created

This commit is contained in:
Karel Krýda
2022-01-23 15:22:00 +01:00
parent c3c4db52ec
commit 0d3414c6d6
32 changed files with 1121 additions and 51 deletions

View File

@@ -53,6 +53,7 @@ class Database {
"patch-2fa-invalidate-used-token.sql": true,
"patch-notification_sent_history.sql": true,
"patch-monitor-basic-auth.sql": true,
"patch-maintenance-table.sql": true,
}
/**

View File

@@ -10,6 +10,7 @@ const { BeanModel } = require("redbean-node/dist/bean-model");
* 0 = DOWN
* 1 = UP
* 2 = PENDING
* 3 = MAINTENANCE
*/
class Heartbeat extends BeanModel {

View File

@@ -0,0 +1,38 @@
const dayjs = require("dayjs");
const utc = require("dayjs/plugin/utc");
let timezone = require("dayjs/plugin/timezone");
dayjs.extend(utc);
dayjs.extend(timezone);
const { BeanModel } = require("redbean-node/dist/bean-model");
class Maintenance extends BeanModel {
/**
* Return a object that ready to parse to JSON for public
* Only show necessary data to public
*/
async toPublicJSON() {
return {
id: this.id,
title: this.title,
description: this.description,
start_date: this.start_date,
end_date: this.end_date
};
}
/**
* Return a object that ready to parse to JSON
*/
async toJSON() {
return {
id: this.id,
title: this.title,
description: this.description,
start_date: this.start_date,
end_date: this.end_date
};
}
}
module.exports = Maintenance;

View File

@@ -6,7 +6,7 @@ dayjs.extend(utc);
dayjs.extend(timezone);
const axios = require("axios");
const { Prometheus } = require("../prometheus");
const { debug, UP, DOWN, PENDING, flipStatus, TimeLogger } = require("../../src/util");
const { debug, UP, DOWN, PENDING, MAINTENANCE, flipStatus, TimeLogger} = require("../../src/util");
const { tcping, ping, dnsResolve, checkCertificate, checkStatusCode, getTotalClientInRoom, setting, errorLog } = require("../util-server");
const { R } = require("redbean-node");
const { BeanModel } = require("redbean-node/dist/bean-model");
@@ -20,6 +20,7 @@ const apicache = require("../modules/apicache");
* 0 = DOWN
* 1 = UP
* 2 = PENDING
* 3 = MAINTENANCE
*/
class Monitor extends BeanModel {
@@ -28,9 +29,12 @@ class Monitor extends BeanModel {
* Only show necessary data to public
*/
async toPublicJSON() {
const maintenance = await R.getAll("SELECT mm.*, maintenance.start_date, maintenance.end_date FROM monitor_maintenance mm JOIN maintenance ON mm.maintenance_id = maintenance.id WHERE mm.monitor_id = ? AND datetime(maintenance.start_date) <= datetime('now', 'localtime') AND datetime(maintenance.end_date) >= datetime('now', 'localtime')", [this.id]);
return {
id: this.id,
name: this.name,
maintenance: (maintenance.length !== 0),
};
}
@@ -50,6 +54,7 @@ class Monitor extends BeanModel {
}
const tags = await R.getAll("SELECT mt.*, tag.name, tag.color FROM monitor_tag mt JOIN tag ON mt.tag_id = tag.id WHERE mt.monitor_id = ?", [this.id]);
const maintenance = await R.getAll("SELECT mm.*, maintenance.start_date, maintenance.end_date FROM monitor_maintenance mm JOIN maintenance ON mm.maintenance_id = maintenance.id WHERE mm.monitor_id = ? AND datetime(maintenance.start_date) <= datetime('now', 'localtime') AND datetime(maintenance.end_date) >= datetime('now', 'localtime')", [this.id]);
return {
id: this.id,
@@ -79,6 +84,7 @@ class Monitor extends BeanModel {
pushToken: this.pushToken,
notificationIDList,
tags: tags,
maintenance: (maintenance.length !== 0),
};
}
@@ -136,6 +142,8 @@ class Monitor extends BeanModel {
bean.time = R.isoDateTime(dayjs.utc());
bean.status = DOWN;
const maintenance = await R.getAll("SELECT mm.*, maintenance.start_date, maintenance.end_date FROM monitor_maintenance mm JOIN maintenance ON mm.maintenance_id = maintenance.id WHERE mm.monitor_id = ? AND datetime(maintenance.start_date) <= datetime('now', 'localtime') AND datetime(maintenance.end_date) >= datetime('now', 'localtime')", [this.id]);
if (this.isUpsideDown()) {
bean.status = flipStatus(bean.status);
}
@@ -148,7 +156,11 @@ class Monitor extends BeanModel {
}
try {
if (this.type === "http" || this.type === "keyword") {
if (maintenance.length !== 0) {
bean.msg = "Monitor under maintenance";
bean.status = MAINTENANCE;
}
else if (this.type === "http" || this.type === "keyword") {
// Do not do any queries/high loading things before the "bean.ping"
let startTime = dayjs().valueOf();
@@ -387,8 +399,13 @@ class Monitor extends BeanModel {
if (isImportant) {
bean.important = true;
debug(`[${this.name}] sendNotification`);
await Monitor.sendNotification(isFirstBeat, this, bean);
if (Monitor.isImportantForNotification(isFirstBeat, previousBeat?.status, bean.status)) {
debug(`[${this.name}] sendNotification`);
await Monitor.sendNotification(isFirstBeat, this, bean);
}
else {
debug(`[${this.name}] will not sendNotification because it is (or was) under maintenance`);
}
// Clear Status Page Cache
debug(`[${this.name}] apicache clear`);
@@ -405,6 +422,8 @@ class Monitor extends BeanModel {
beatInterval = this.retryInterval;
}
console.warn(`Monitor #${this.id} '${this.name}': Pending: ${bean.msg} | Max retries: ${this.maxretries} | Retry: ${retries} | Retry Interval: ${beatInterval} seconds | Type: ${this.type}`);
} else if (bean.status === MAINTENANCE) {
console.warn(`Monitor #${this.id} '${this.name}': Under Maintenance | Type: ${this.type}`);
} else {
console.warn(`Monitor #${this.id} '${this.name}': Failing: ${bean.msg} | Interval: ${beatInterval} seconds | Type: ${this.type}`);
}
@@ -598,7 +617,7 @@ class Monitor extends BeanModel {
-- SUM all uptime duration, also trim off the beat out of time window
SUM(
CASE
WHEN (status = 1)
WHEN (status = 1 OR status = 3)
THEN
CASE
WHEN (JULIANDAY(\`time\`) - JULIANDAY(?)) * 86400 < duration
@@ -659,11 +678,42 @@ class Monitor extends BeanModel {
// DOWN -> PENDING = this case not exists
// DOWN -> DOWN = not important
// * DOWN -> UP = important
let isImportant = isFirstBeat ||
// MAINTENANCE -> MAINTENANCE = not important
// * MAINTENANCE -> UP = important
// * MAINTENANCE -> DOWN = important
// * DOWN -> MAINTENANCE = important
// * UP -> MAINTENANCE = important
return isFirstBeat ||
(previousBeatStatus === DOWN && currentBeatStatus === MAINTENANCE) ||
(previousBeatStatus === UP && currentBeatStatus === MAINTENANCE) ||
(previousBeatStatus === MAINTENANCE && currentBeatStatus === DOWN) ||
(previousBeatStatus === MAINTENANCE && currentBeatStatus === UP) ||
(previousBeatStatus === UP && currentBeatStatus === DOWN) ||
(previousBeatStatus === DOWN && currentBeatStatus === UP) ||
(previousBeatStatus === PENDING && currentBeatStatus === DOWN);
}
static isImportantForNotification(isFirstBeat, previousBeatStatus, currentBeatStatus) {
// * ? -> ANY STATUS = important [isFirstBeat]
// UP -> PENDING = not important
// * UP -> DOWN = important
// UP -> UP = not important
// PENDING -> PENDING = not important
// * PENDING -> DOWN = important
// PENDING -> UP = not important
// DOWN -> PENDING = this case not exists
// DOWN -> DOWN = not important
// * DOWN -> UP = important
// MAINTENANCE -> MAINTENANCE = not important
// MAINTENANCE -> UP = not important
// * MAINTENANCE -> DOWN = important
// DOWN -> MAINTENANCE = not important
// UP -> MAINTENANCE = not important
return isFirstBeat ||
(previousBeatStatus === MAINTENANCE && currentBeatStatus === DOWN) ||
(previousBeatStatus === UP && currentBeatStatus === DOWN) ||
(previousBeatStatus === DOWN && currentBeatStatus === UP) ||
(previousBeatStatus === PENDING && currentBeatStatus === DOWN);
return isImportant;
}
static async sendNotification(isFirstBeat, monitor, bean) {

View File

@@ -5,7 +5,7 @@ const server = require("../server");
const apicache = require("../modules/apicache");
const Monitor = require("../model/monitor");
const dayjs = require("dayjs");
const { UP, flipStatus, debug } = require("../../src/util");
const { UP, MAINTENANCE, flipStatus, debug} = require("../../src/util");
let router = express.Router();
let cache = apicache.middleware;
@@ -51,6 +51,12 @@ router.get("/api/push/:pushToken", async (request, response) => {
duration = dayjs(bean.time).diff(dayjs(previousHeartbeat.time), "second");
}
const maintenance = await R.getAll("SELECT mm.*, maintenance.start_date, maintenance.end_date FROM monitor_maintenance mm JOIN maintenance ON mm.maintenance_id = maintenance.id WHERE mm.monitor_id = ? AND datetime(maintenance.start_date) <= datetime('now', 'localtime') AND datetime(maintenance.end_date) >= datetime('now', 'localtime')", [monitor.id]);
if (maintenance.length !== 0) {
msg = "Monitor under maintenance";
status = MAINTENANCE;
}
debug("PreviousStatus: " + previousStatus);
debug("Current Status: " + status);
@@ -70,7 +76,7 @@ router.get("/api/push/:pushToken", async (request, response) => {
ok: true,
});
if (bean.important) {
if (Monitor.isImportantForNotification(isFirstBeat, previousStatus, status)) {
await Monitor.sendNotification(isFirstBeat, monitor, bean);
}
@@ -131,6 +137,34 @@ router.get("/api/status-page/incident", async (_, response) => {
}
});
// Status Page - Maintenance List
// Can fetch only if published
router.get("/api/status-page/maintenance-list", async (_request, response) => {
allowDevAllOrigin(response);
try {
await checkPublished();
const publicMaintenanceList = [];
let maintenanceBeanList = R.convertToBeans("maintenance", await R.getAll(`
SELECT maintenance.*
FROM maintenance
WHERE datetime(maintenance.start_date) <= datetime('now', 'localtime')
AND datetime(maintenance.end_date) >= datetime('now', 'localtime')
ORDER BY maintenance.end_date
`));
for (const bean of maintenanceBeanList) {
publicMaintenanceList.push(await bean.toPublicJSON());
}
response.json(publicMaintenanceList);
} catch (error) {
send403(response, error.message);
}
});
// Status Page - Monitor List
// Can fetch only if published
router.get("/api/status-page/monitor-list", cache("5 minutes"), async (_request, response) => {

View File

@@ -132,6 +132,7 @@ const { sendNotificationList, sendHeartbeatList, sendImportantHeartbeatList, sen
const { statusPageSocketHandler } = require("./socket-handlers/status-page-socket-handler");
const databaseSocketHandler = require("./socket-handlers/database-socket-handler");
const TwoFA = require("./2fa");
const apicache = require("./modules/apicache");
app.use(express.json());
@@ -162,6 +163,12 @@ let jwtSecret = null;
*/
let monitorList = {};
/**
* Main maintenance list
* @type {{}}
*/
let maintenanceList = {};
/**
* Show Setup Page
* @type {boolean}
@@ -625,6 +632,101 @@ exports.entryPage = "dashboard";
}
});
// Add a new maintenance
socket.on("addMaintenance", async (maintenance, callback) => {
try {
checkLogin(socket);
let bean = R.dispense("maintenance");
bean.import(maintenance);
bean.user_id = socket.userID;
let maintenanceID = await R.store(bean);
await sendMaintenanceList(socket);
callback({
ok: true,
msg: "Added Successfully.",
maintenanceID,
});
} catch (e) {
callback({
ok: false,
msg: e.message,
});
}
});
// Edit a maintenance
socket.on("editMaintenance", async (maintenance, callback) => {
try {
checkLogin(socket);
let bean = await R.findOne("maintenance", " id = ? ", [ maintenance.id ]);
if (bean.user_id !== socket.userID) {
throw new Error("Permission denied.");
}
bean.title = maintenance.title;
bean.description = maintenance.description;
bean.start_date = maintenance.start_date;
bean.end_date = maintenance.end_date;
await R.store(bean);
await sendMaintenanceList(socket);
callback({
ok: true,
msg: "Saved.",
maintenanceID: bean.id,
});
} catch (e) {
console.error(e);
callback({
ok: false,
msg: e.message,
});
}
});
// Add a new monitor_maintenance
socket.on("addMonitorMaintenance", async (maintenanceID, monitors, callback) => {
try {
checkLogin(socket);
await R.exec("DELETE FROM monitor_maintenance WHERE maintenance_id = ?", [
maintenanceID
]);
for await (const monitor of monitors) {
let bean = R.dispense("monitor_maintenance");
bean.import({
monitor_id: monitor.id,
maintenance_id: maintenanceID
});
await R.store(bean);
}
apicache.clear();
callback({
ok: true,
msg: "Added Successfully.",
});
} catch (e) {
callback({
ok: false,
msg: e.message,
});
}
});
socket.on("getMonitorList", async (callback) => {
try {
checkLogin(socket);
@@ -641,6 +743,22 @@ exports.entryPage = "dashboard";
}
});
socket.on("getMaintenanceList", async (callback) => {
try {
checkLogin(socket);
await sendMaintenanceList(socket);
callback({
ok: true,
});
} catch (e) {
console.error(e);
callback({
ok: false,
msg: e.message,
});
}
});
socket.on("getMonitor", async (monitorID, callback) => {
try {
checkLogin(socket);
@@ -665,6 +783,54 @@ exports.entryPage = "dashboard";
}
});
socket.on("getMaintenance", async (maintenanceID, callback) => {
try {
checkLogin(socket);
console.log(`Get Maintenance: ${maintenanceID} User ID: ${socket.userID}`);
let bean = await R.findOne("maintenance", " id = ? AND user_id = ? ", [
maintenanceID,
socket.userID,
]);
callback({
ok: true,
maintenance: await bean.toJSON(),
});
} catch (e) {
callback({
ok: false,
msg: e.message,
});
}
});
socket.on("getMonitorMaintenance", async (maintenanceID, callback) => {
try {
checkLogin(socket);
console.log(`Get Monitors for Maintenance: ${maintenanceID} User ID: ${socket.userID}`);
let monitors = await R.getAll("SELECT monitor.id, monitor.name FROM monitor_maintenance mm JOIN monitor ON mm.monitor_id = monitor.id WHERE mm.maintenance_id = ? ", [
maintenanceID,
]);
callback({
ok: true,
monitors,
});
} catch (e) {
console.error(e);
callback({
ok: false,
msg: e.message,
});
}
});
socket.on("getMonitorBeats", async (monitorID, period, callback) => {
try {
checkLogin(socket);
@@ -769,6 +935,36 @@ exports.entryPage = "dashboard";
}
});
socket.on("deleteMaintenance", async (maintenanceID, callback) => {
try {
checkLogin(socket);
console.log(`Delete Maintenance: ${maintenanceID} User ID: ${socket.userID}`);
if (maintenanceID in maintenanceList) {
delete maintenanceList[maintenanceID];
}
await R.exec("DELETE FROM maintenance WHERE id = ? AND user_id = ? ", [
maintenanceID,
socket.userID,
]);
callback({
ok: true,
msg: "Deleted Successfully.",
});
await sendMaintenanceList(socket);
} catch (e) {
callback({
ok: false,
msg: e.message,
});
}
});
socket.on("getTags", async (callback) => {
try {
checkLogin(socket);
@@ -1394,11 +1590,18 @@ async function sendMonitorList(socket) {
return list;
}
async function sendMaintenanceList(socket) {
let list = await getMaintenanceJSONList(socket.userID);
io.to(socket.userID).emit("maintenanceList", list);
return list;
}
async function afterLogin(socket, user) {
socket.userID = user.id;
socket.join(user.id);
let monitorList = await sendMonitorList(socket);
sendMaintenanceList(socket);
sendNotificationList(socket);
await sleep(500);
@@ -1430,6 +1633,20 @@ async function getMonitorJSONList(userID) {
return result;
}
async function getMaintenanceJSONList(userID) {
let result = {};
let maintenanceList = await R.find("maintenance", " user_id = ? ORDER BY end_date DESC, title", [
userID,
]);
for (let maintenance of maintenanceList) {
result[maintenance.id] = await maintenance.toJSON();
}
return result;
}
async function initDatabase(testMode = false) {
if (! fs.existsSync(Database.path)) {
console.log("Copying Database");