Clean up data

This commit is contained in:
Louis Lam
2024-10-26 15:37:50 +08:00
parent e91b2efe9a
commit 65b49384e0
3 changed files with 55 additions and 31 deletions

View File

@@ -1,35 +1,19 @@
const { R } = require("redbean-node");
const { log } = require("../../src/util");
const { setSetting, setting } = require("../util-server");
const { setSetting } = require("../util-server");
const Database = require("../database");
const { Settings } = require("../settings");
const dayjs = require("dayjs");
const DEFAULT_KEEP_PERIOD = 180;
const DEFAULT_KEEP_PERIOD = 365;
/**
* Clears old data from the heartbeat table of the database.
* Clears old data from the heartbeat table and the stat_daily of the database.
* @returns {Promise<void>} A promise that resolves when the data has been cleared.
*/
const clearOldData = async () => {
// TODO: Temporary disable for testing
return;
/*
* TODO:
* Since we have aggregated table now, we don't need so much data in heartbeat table.
* But we still need to keep the important rows, because they contain the message.
*
* In the heartbeat table:
* - important rows: keep according to the setting (keepDataPeriodDays) (default 180 days)
* - not important rows: keep 2 days
*
* stat_* tables:
* - keep according to the setting (keepDataPeriodDays) (default 180 days)
*/
let period = await setting("keepDataPeriodDays");
await Database.clearHeartbeatData();
let period = await Settings.get("keepDataPeriodDays");
// Set Default Period
if (period == null) {
@@ -50,16 +34,21 @@ const clearOldData = async () => {
if (parsedPeriod < 1) {
log.info("clearOldData", `Data deletion has been disabled as period is less than 1. Period is ${parsedPeriod} days.`);
} else {
log.debug("clearOldData", `Clearing Data older than ${parsedPeriod} days...`);
const sqlHourOffset = Database.sqlHourOffset();
try {
await R.exec(
"DELETE FROM heartbeat WHERE time < " + sqlHourOffset,
[ parsedPeriod * -24 ]
);
// Heartbeat
await R.exec("DELETE FROM heartbeat WHERE time < " + sqlHourOffset, [
parsedPeriod * -24,
]);
let timestamp = dayjs().subtract(parsedPeriod, "day").utc().startOf("day").unix();
// stat_daily
await R.exec("DELETE FROM stat_daily WHERE timestamp < ? ", [
timestamp,
]);
if (Database.dbConfig.type === "sqlite") {
await R.exec("PRAGMA optimize;");
@@ -68,6 +57,8 @@ const clearOldData = async () => {
log.error("clearOldData", `Failed to clear old data: ${e.message}`);
}
}
log.debug("clearOldData", "Data cleared.");
};
module.exports = {