feat: rebuild xiteng.site homelab platform
This commit is contained in:
@@ -0,0 +1,852 @@
|
||||
import http from "node:http";
|
||||
import {mkdirSync} from "node:fs";
|
||||
import {DatabaseSync} from "node:sqlite";
|
||||
|
||||
const port = Number.parseInt(process.env.PORT || "8091", 10);
|
||||
const dockerSocket = process.env.DOCKER_SOCKET || "/var/run/docker.sock";
|
||||
const databasePath = process.env.DATABASE_PATH || "/data/registry.db";
|
||||
const discoveryIntervalMs = boundedInteger(process.env.DISCOVERY_INTERVAL_MS, 5000, 2000, 60000);
|
||||
const missingRetentionDays = boundedInteger(process.env.MISSING_RETENTION_DAYS, 30, 1, 3650);
|
||||
const rawRetentionDays = boundedInteger(process.env.RAW_RETENTION_DAYS, 30, 1, 3650);
|
||||
const aggregateRetentionDays = boundedInteger(process.env.AGGREGATE_RETENTION_DAYS, 365, 30, 3650);
|
||||
const monitorConcurrency = boundedInteger(process.env.MONITOR_CONCURRENCY, 4, 1, 32);
|
||||
const adminProxyHeader = process.env.ADMIN_PROXY_HEADER || "X-Portal-Admin";
|
||||
const labelPrefix = "xiteng.site.component.";
|
||||
|
||||
const allowedFields = new Set([
|
||||
"enabled",
|
||||
"name",
|
||||
"description",
|
||||
"section",
|
||||
"category",
|
||||
"url",
|
||||
"endpoint",
|
||||
"access",
|
||||
"access-label",
|
||||
"icon",
|
||||
"icon-url",
|
||||
"navigation",
|
||||
"portal-link",
|
||||
"accent",
|
||||
"order",
|
||||
"monitor.enabled",
|
||||
"monitor.url",
|
||||
"monitor.interval",
|
||||
"monitor.failures",
|
||||
"monitor.timeout",
|
||||
"monitor.accept"
|
||||
]);
|
||||
|
||||
const allowedSections = new Set(["services", "infrastructure"]);
|
||||
const allowedAccents = new Set(["red", "green", "yellow", "blue", "ink"]);
|
||||
const activeChecks = new Set();
|
||||
let discoveryRunning = false;
|
||||
let lastDiscoveryAt = null;
|
||||
|
||||
mkdirSync(databasePath.slice(0, databasePath.lastIndexOf("/")) || ".", {recursive: true});
|
||||
const database = new DatabaseSync(databasePath);
|
||||
database.exec(`
|
||||
PRAGMA journal_mode = WAL;
|
||||
PRAGMA foreign_keys = ON;
|
||||
PRAGMA busy_timeout = 5000;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS registry_component (
|
||||
id TEXT PRIMARY KEY,
|
||||
first_seen TEXT NOT NULL,
|
||||
last_seen TEXT NOT NULL,
|
||||
lifecycle TEXT NOT NULL DEFAULT 'active',
|
||||
missing_since TEXT,
|
||||
archived_at TEXT,
|
||||
archive_reason TEXT,
|
||||
metadata_json TEXT NOT NULL,
|
||||
monitor_json TEXT,
|
||||
monitor_state TEXT NOT NULL DEFAULT 'unmonitored',
|
||||
monitor_paused INTEGER NOT NULL DEFAULT 0,
|
||||
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
||||
last_checked_at TEXT,
|
||||
last_status_code INTEGER,
|
||||
last_latency_ms INTEGER,
|
||||
last_error TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS monitor_check (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
component_id TEXT NOT NULL REFERENCES registry_component(id) ON DELETE CASCADE,
|
||||
checked_at TEXT NOT NULL,
|
||||
ok INTEGER NOT NULL,
|
||||
status_code INTEGER,
|
||||
latency_ms INTEGER,
|
||||
error TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS monitor_check_component_time
|
||||
ON monitor_check(component_id, checked_at);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS monitor_check_unique
|
||||
ON monitor_check(component_id, checked_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS monitor_incident (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
component_id TEXT NOT NULL REFERENCES registry_component(id) ON DELETE CASCADE,
|
||||
started_at TEXT NOT NULL,
|
||||
ended_at TEXT,
|
||||
cause TEXT
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS monitor_incident_open
|
||||
ON monitor_incident(component_id) WHERE ended_at IS NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS monitor_hourly (
|
||||
component_id TEXT NOT NULL REFERENCES registry_component(id) ON DELETE CASCADE,
|
||||
bucket TEXT NOT NULL,
|
||||
total INTEGER NOT NULL DEFAULT 0,
|
||||
successful INTEGER NOT NULL DEFAULT 0,
|
||||
latency_sum INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (component_id, bucket)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS monitor_daily (
|
||||
component_id TEXT NOT NULL REFERENCES registry_component(id) ON DELETE CASCADE,
|
||||
bucket TEXT NOT NULL,
|
||||
total INTEGER NOT NULL DEFAULT 0,
|
||||
successful INTEGER NOT NULL DEFAULT 0,
|
||||
latency_sum INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (component_id, bucket)
|
||||
);
|
||||
`);
|
||||
|
||||
const selectComponent = database.prepare("SELECT * FROM registry_component WHERE id = ?");
|
||||
const selectComponents = database.prepare("SELECT * FROM registry_component ORDER BY id");
|
||||
const insertComponent = database.prepare(`
|
||||
INSERT INTO registry_component (
|
||||
id, first_seen, last_seen, lifecycle, metadata_json, monitor_json, monitor_state
|
||||
) VALUES (?, ?, ?, 'active', ?, ?, ?)
|
||||
`);
|
||||
const updateDiscoveredComponent = database.prepare(`
|
||||
UPDATE registry_component
|
||||
SET last_seen = ?,
|
||||
lifecycle = ?,
|
||||
missing_since = NULL,
|
||||
archived_at = CASE WHEN archive_reason = 'manual' THEN archived_at ELSE NULL END,
|
||||
archive_reason = CASE WHEN archive_reason = 'manual' THEN archive_reason ELSE NULL END,
|
||||
metadata_json = ?,
|
||||
monitor_json = ?,
|
||||
monitor_state = ?,
|
||||
monitor_paused = ?
|
||||
WHERE id = ?
|
||||
`);
|
||||
const markMissing = database.prepare(`
|
||||
UPDATE registry_component
|
||||
SET lifecycle = 'missing', missing_since = ?
|
||||
WHERE id = ? AND lifecycle = 'active'
|
||||
`);
|
||||
const markArchivedByRetention = database.prepare(`
|
||||
UPDATE registry_component
|
||||
SET lifecycle = 'archived', archived_at = ?, archive_reason = 'retention',
|
||||
monitor_state = CASE WHEN monitor_json IS NULL THEN 'unmonitored' ELSE 'paused' END,
|
||||
monitor_paused = CASE WHEN monitor_json IS NULL THEN 0 ELSE 1 END
|
||||
WHERE id = ? AND lifecycle = 'missing'
|
||||
`);
|
||||
const insertCheck = database.prepare(`
|
||||
INSERT INTO monitor_check (component_id, checked_at, ok, status_code, latency_ms, error)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
const upsertHourly = database.prepare(`
|
||||
INSERT INTO monitor_hourly (component_id, bucket, total, successful, latency_sum)
|
||||
VALUES (?, ?, 1, ?, ?)
|
||||
ON CONFLICT(component_id, bucket) DO UPDATE SET
|
||||
total = total + 1,
|
||||
successful = successful + excluded.successful,
|
||||
latency_sum = latency_sum + excluded.latency_sum
|
||||
`);
|
||||
const upsertDaily = database.prepare(`
|
||||
INSERT INTO monitor_daily (component_id, bucket, total, successful, latency_sum)
|
||||
VALUES (?, ?, 1, ?, ?)
|
||||
ON CONFLICT(component_id, bucket) DO UPDATE SET
|
||||
total = total + 1,
|
||||
successful = successful + excluded.successful,
|
||||
latency_sum = latency_sum + excluded.latency_sum
|
||||
`);
|
||||
const uptimeSince = database.prepare(`
|
||||
SELECT COUNT(*) AS total, COALESCE(SUM(ok), 0) AS successful
|
||||
FROM monitor_check
|
||||
WHERE component_id = ? AND checked_at >= ?
|
||||
`);
|
||||
const openIncident = database.prepare(`
|
||||
SELECT started_at FROM monitor_incident
|
||||
WHERE component_id = ? AND ended_at IS NULL
|
||||
`);
|
||||
|
||||
function boundedInteger(value, fallback, minimum, maximum) {
|
||||
const parsed = Number.parseInt(value || "", 10);
|
||||
if (!Number.isFinite(parsed)) {
|
||||
return fallback;
|
||||
}
|
||||
return Math.max(minimum, Math.min(maximum, parsed));
|
||||
}
|
||||
|
||||
function dockerGet(pathname) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = http.request({
|
||||
socketPath: dockerSocket,
|
||||
path: pathname,
|
||||
method: "GET",
|
||||
headers: {"Accept": "application/json"}
|
||||
}, (response) => {
|
||||
const chunks = [];
|
||||
response.on("data", (chunk) => chunks.push(chunk));
|
||||
response.on("end", () => {
|
||||
const body = Buffer.concat(chunks).toString("utf8");
|
||||
if ((response.statusCode || 500) >= 400) {
|
||||
reject(new Error(`Docker API ${response.statusCode}`));
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
resolve(JSON.parse(body));
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
request.setTimeout(3000, () => request.destroy(new Error("Docker API timeout")));
|
||||
request.on("error", reject);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
function limitedString(value, fallback = "", limit = 500) {
|
||||
return typeof value === "string" ? value.trim().slice(0, limit) : fallback;
|
||||
}
|
||||
|
||||
function safeUrl(value) {
|
||||
const candidate = limitedString(value);
|
||||
if (!candidate) {
|
||||
return "";
|
||||
}
|
||||
|
||||
try {
|
||||
const url = new URL(candidate);
|
||||
if (url.username || url.password) {
|
||||
return "";
|
||||
}
|
||||
return ["http:", "https:"].includes(url.protocol) ? url.href : "";
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function healthFromStatus(status) {
|
||||
if (status.includes("(unhealthy)")) {
|
||||
return "unhealthy";
|
||||
}
|
||||
if (status.includes("(healthy)")) {
|
||||
return "healthy";
|
||||
}
|
||||
if (status.includes("(health: starting)")) {
|
||||
return "starting";
|
||||
}
|
||||
return "none";
|
||||
}
|
||||
|
||||
function instanceState(container) {
|
||||
const state = limitedString(container.State, "unknown").toLowerCase();
|
||||
const health = healthFromStatus(limitedString(container.Status));
|
||||
return health === "unhealthy" ? "unhealthy" : state;
|
||||
}
|
||||
|
||||
function aggregateStatus(instances) {
|
||||
const states = instances.map((instance) => instance.state);
|
||||
if (states.length > 0 && states.every((state) => state === "running")) {
|
||||
return "running";
|
||||
}
|
||||
if (states.includes("unhealthy")) {
|
||||
return "unhealthy";
|
||||
}
|
||||
if (states.includes("restarting")) {
|
||||
return "restarting";
|
||||
}
|
||||
if (states.includes("running")) {
|
||||
return "degraded";
|
||||
}
|
||||
for (const state of ["dead", "exited", "paused", "created"]) {
|
||||
if (states.includes(state)) {
|
||||
return state;
|
||||
}
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
function definitionsFromLabels(labels) {
|
||||
const definitions = new Map();
|
||||
|
||||
for (const [key, value] of Object.entries(labels || {})) {
|
||||
if (!key.startsWith(labelPrefix)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const remainder = key.slice(labelPrefix.length);
|
||||
const separator = remainder.indexOf(".");
|
||||
if (separator <= 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const id = remainder.slice(0, separator);
|
||||
const field = remainder.slice(separator + 1);
|
||||
if (!/^[a-z0-9][a-z0-9-]*$/.test(id) || !allowedFields.has(field)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!definitions.has(id)) {
|
||||
definitions.set(id, {});
|
||||
}
|
||||
definitions.get(id)[field] = limitedString(value);
|
||||
}
|
||||
|
||||
return definitions;
|
||||
}
|
||||
|
||||
function publicDefinition(id, definition) {
|
||||
const order = Number.parseInt(definition.order || "999", 10);
|
||||
return {
|
||||
id,
|
||||
name: limitedString(definition.name, id),
|
||||
description: limitedString(definition.description),
|
||||
section: allowedSections.has(definition.section) ? definition.section : "infrastructure",
|
||||
category: limitedString(definition.category, "其他"),
|
||||
url: safeUrl(definition.url),
|
||||
endpoint: limitedString(definition.endpoint, "", 160),
|
||||
access: limitedString(definition.access, "internal", 40),
|
||||
accessLabel: limitedString(definition["access-label"], "受控访问", 100),
|
||||
icon: limitedString(definition.icon, id.slice(0, 2).toUpperCase(), 8),
|
||||
iconUrl: safeUrl(definition["icon-url"]),
|
||||
navigation: ["new-tab", "same-tab", "endpoint"].includes(definition.navigation) ? definition.navigation : (definition.url ? "new-tab" : "endpoint"),
|
||||
portalLink: ["embedded", "native", "none"].includes(definition["portal-link"]) ? definition["portal-link"] : "none",
|
||||
accent: allowedAccents.has(definition.accent) ? definition.accent : "ink",
|
||||
order: Number.isFinite(order) ? order : 999
|
||||
};
|
||||
}
|
||||
|
||||
function monitorDefinition(definition) {
|
||||
if (definition["monitor.enabled"]?.toLowerCase() !== "true") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const url = safeUrl(definition["monitor.url"]);
|
||||
if (!url) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
type: "http",
|
||||
url,
|
||||
interval: boundedInteger(definition["monitor.interval"], 60, 10, 86400),
|
||||
failures: boundedInteger(definition["monitor.failures"], 3, 1, 20),
|
||||
timeout: boundedInteger(definition["monitor.timeout"], 10, 1, 60),
|
||||
accept: limitedString(definition["monitor.accept"], "200-299", 80)
|
||||
};
|
||||
}
|
||||
|
||||
function parseJson(value, fallback) {
|
||||
try {
|
||||
return value ? JSON.parse(value) : fallback;
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
async function discoverDockerComponents() {
|
||||
const containers = await dockerGet("/containers/json?all=true");
|
||||
const components = new Map();
|
||||
|
||||
for (const container of containers) {
|
||||
const labels = container.Labels || {};
|
||||
for (const [id, definition] of definitionsFromLabels(labels)) {
|
||||
if (definition.enabled?.toLowerCase() !== "true") {
|
||||
continue;
|
||||
}
|
||||
|
||||
const metadata = publicDefinition(id, definition);
|
||||
const instance = {
|
||||
container: limitedString(container.Names?.[0] || "", "", 160).replace(/^\//, ""),
|
||||
service: limitedString(labels["com.docker.compose.service"], "docker", 160),
|
||||
project: limitedString(labels["com.docker.compose.project"], "", 160),
|
||||
image: limitedString(container.Image, "", 300),
|
||||
state: instanceState(container),
|
||||
health: healthFromStatus(limitedString(container.Status))
|
||||
};
|
||||
|
||||
if (!components.has(id)) {
|
||||
components.set(id, {...metadata, monitor: monitorDefinition(definition), instances: []});
|
||||
}
|
||||
components.get(id).instances.push(instance);
|
||||
}
|
||||
}
|
||||
|
||||
for (const component of components.values()) {
|
||||
const primary = component.instances[0];
|
||||
component.status = aggregateStatus(component.instances);
|
||||
component.instanceCount = component.instances.length;
|
||||
component.container = primary.container;
|
||||
component.service = primary.service;
|
||||
component.project = primary.project;
|
||||
component.image = primary.image;
|
||||
}
|
||||
|
||||
return components;
|
||||
}
|
||||
|
||||
function reconcileComponent(id, discovered, now) {
|
||||
const existing = selectComponent.get(id);
|
||||
const metadata = {...discovered};
|
||||
delete metadata.monitor;
|
||||
delete metadata.instances;
|
||||
const metadataJson = JSON.stringify(metadata);
|
||||
const monitorJson = discovered.monitor ? JSON.stringify(discovered.monitor) : null;
|
||||
|
||||
if (!existing) {
|
||||
insertComponent.run(
|
||||
id,
|
||||
now,
|
||||
now,
|
||||
metadataJson,
|
||||
monitorJson,
|
||||
monitorJson ? "pending" : "unmonitored"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const manuallyArchived = existing.lifecycle === "archived" && existing.archive_reason === "manual";
|
||||
const monitorChanged = existing.monitor_json !== monitorJson;
|
||||
let monitorState = existing.monitor_state;
|
||||
let monitorPaused = existing.monitor_paused;
|
||||
|
||||
if (!monitorJson) {
|
||||
monitorState = "unmonitored";
|
||||
monitorPaused = 0;
|
||||
} else if (manuallyArchived) {
|
||||
monitorState = "paused";
|
||||
monitorPaused = 1;
|
||||
} else if (monitorChanged || existing.lifecycle === "archived") {
|
||||
monitorState = "pending";
|
||||
monitorPaused = 0;
|
||||
}
|
||||
|
||||
updateDiscoveredComponent.run(
|
||||
now,
|
||||
manuallyArchived ? "archived" : "active",
|
||||
metadataJson,
|
||||
monitorJson,
|
||||
monitorState,
|
||||
monitorPaused,
|
||||
id
|
||||
);
|
||||
|
||||
if (monitorChanged) {
|
||||
database.prepare(`
|
||||
UPDATE registry_component
|
||||
SET consecutive_failures = 0, last_checked_at = NULL, last_status_code = NULL,
|
||||
last_latency_ms = NULL, last_error = NULL
|
||||
WHERE id = ?
|
||||
`).run(id);
|
||||
database.prepare(`
|
||||
UPDATE monitor_incident SET ended_at = ?
|
||||
WHERE component_id = ? AND ended_at IS NULL
|
||||
`).run(now, id);
|
||||
}
|
||||
}
|
||||
|
||||
async function reconcileDiscovery() {
|
||||
if (discoveryRunning) {
|
||||
return;
|
||||
}
|
||||
discoveryRunning = true;
|
||||
|
||||
try {
|
||||
const now = new Date().toISOString();
|
||||
const discovered = await discoverDockerComponents();
|
||||
|
||||
database.exec("BEGIN IMMEDIATE");
|
||||
try {
|
||||
for (const [id, component] of discovered) {
|
||||
reconcileComponent(id, component, now);
|
||||
}
|
||||
|
||||
for (const existing of selectComponents.all()) {
|
||||
if (discovered.has(existing.id) || existing.lifecycle === "archived") {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (existing.lifecycle === "active") {
|
||||
markMissing.run(now, existing.id);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (existing.lifecycle === "missing" && existing.missing_since) {
|
||||
const missingForMs = Date.now() - new Date(existing.missing_since).getTime();
|
||||
if (missingForMs >= missingRetentionDays * 86400000) {
|
||||
markArchivedByRetention.run(now, existing.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
database.exec("COMMIT");
|
||||
} catch (error) {
|
||||
database.exec("ROLLBACK");
|
||||
throw error;
|
||||
}
|
||||
|
||||
lastDiscoveryAt = now;
|
||||
} catch (error) {
|
||||
console.error("Docker discovery failed", error.message);
|
||||
} finally {
|
||||
discoveryRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
function statusAccepted(statusCode, expression) {
|
||||
for (const part of expression.split(",")) {
|
||||
const match = part.trim().match(/^(\d{3})(?:-(\d{3}))?$/);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
const minimum = Number.parseInt(match[1], 10);
|
||||
const maximum = Number.parseInt(match[2] || match[1], 10);
|
||||
if (statusCode >= minimum && statusCode <= maximum) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function safeError(error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return limitedString(message.replace(/[\r\n\t]+/g, " "), "probe failed", 300);
|
||||
}
|
||||
|
||||
function updateAggregate(componentId, checkedAt, ok, latencyMs) {
|
||||
const hourBucket = `${checkedAt.slice(0, 13)}:00:00.000Z`;
|
||||
const dayBucket = `${checkedAt.slice(0, 10)}T00:00:00.000Z`;
|
||||
upsertHourly.run(componentId, hourBucket, ok ? 1 : 0, latencyMs || 0);
|
||||
upsertDaily.run(componentId, dayBucket, ok ? 1 : 0, latencyMs || 0);
|
||||
}
|
||||
|
||||
function recordCheck(componentId, monitor, result) {
|
||||
const existing = selectComponent.get(componentId);
|
||||
if (!existing) {
|
||||
return;
|
||||
}
|
||||
|
||||
const nextFailures = result.ok ? 0 : existing.consecutive_failures + 1;
|
||||
const nextState = result.ok ? "up" : nextFailures >= monitor.failures ? "down" : "degraded";
|
||||
|
||||
database.exec("BEGIN IMMEDIATE");
|
||||
try {
|
||||
insertCheck.run(
|
||||
componentId,
|
||||
result.checkedAt,
|
||||
result.ok ? 1 : 0,
|
||||
result.statusCode,
|
||||
result.latencyMs,
|
||||
result.error
|
||||
);
|
||||
updateAggregate(componentId, result.checkedAt, result.ok, result.latencyMs);
|
||||
|
||||
database.prepare(`
|
||||
UPDATE registry_component
|
||||
SET monitor_state = ?, consecutive_failures = ?, last_checked_at = ?,
|
||||
last_status_code = ?, last_latency_ms = ?, last_error = ?
|
||||
WHERE id = ?
|
||||
`).run(
|
||||
nextState,
|
||||
nextFailures,
|
||||
result.checkedAt,
|
||||
result.statusCode,
|
||||
result.latencyMs,
|
||||
result.error,
|
||||
componentId
|
||||
);
|
||||
|
||||
if (nextState === "down" && existing.monitor_state !== "down") {
|
||||
database.prepare(`
|
||||
INSERT OR IGNORE INTO monitor_incident (component_id, started_at, cause)
|
||||
VALUES (?, ?, ?)
|
||||
`).run(componentId, result.checkedAt, result.error || `HTTP ${result.statusCode || "error"}`);
|
||||
}
|
||||
|
||||
if (result.ok) {
|
||||
database.prepare(`
|
||||
UPDATE monitor_incident SET ended_at = ?
|
||||
WHERE component_id = ? AND ended_at IS NULL
|
||||
`).run(result.checkedAt, componentId);
|
||||
}
|
||||
database.exec("COMMIT");
|
||||
} catch (error) {
|
||||
database.exec("ROLLBACK");
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function checkComponent(row, monitor) {
|
||||
const checkedAt = new Date().toISOString();
|
||||
const startedAt = performance.now();
|
||||
let statusCode = null;
|
||||
let ok = false;
|
||||
let errorMessage = null;
|
||||
|
||||
try {
|
||||
const response = await fetch(monitor.url, {
|
||||
method: "GET",
|
||||
redirect: "follow",
|
||||
headers: {
|
||||
"Accept": "text/html,application/json;q=0.9,*/*;q=0.1",
|
||||
"User-Agent": "Xiteng-Portal-Monitor/1.0"
|
||||
},
|
||||
signal: AbortSignal.timeout(monitor.timeout * 1000)
|
||||
});
|
||||
statusCode = response.status;
|
||||
ok = statusAccepted(statusCode, monitor.accept);
|
||||
if (!ok) {
|
||||
errorMessage = `HTTP ${statusCode}`;
|
||||
}
|
||||
if (response.body) {
|
||||
await response.body.cancel();
|
||||
}
|
||||
} catch (error) {
|
||||
errorMessage = safeError(error);
|
||||
}
|
||||
|
||||
const latencyMs = Math.max(0, Math.round(performance.now() - startedAt));
|
||||
recordCheck(row.id, monitor, {
|
||||
checkedAt,
|
||||
ok,
|
||||
statusCode,
|
||||
latencyMs,
|
||||
error: errorMessage
|
||||
});
|
||||
}
|
||||
|
||||
function scheduleDueChecks() {
|
||||
if (activeChecks.size >= monitorConcurrency) {
|
||||
return;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const candidates = selectComponents.all().filter((row) => {
|
||||
if (!row.monitor_json || row.monitor_paused || !["active", "missing"].includes(row.lifecycle)) {
|
||||
return false;
|
||||
}
|
||||
const monitor = parseJson(row.monitor_json, null);
|
||||
if (!monitor) {
|
||||
return false;
|
||||
}
|
||||
const lastChecked = row.last_checked_at ? new Date(row.last_checked_at).getTime() : 0;
|
||||
return !activeChecks.has(row.id) && now - lastChecked >= monitor.interval * 1000;
|
||||
});
|
||||
|
||||
for (const row of candidates.slice(0, monitorConcurrency - activeChecks.size)) {
|
||||
const monitor = parseJson(row.monitor_json, null);
|
||||
activeChecks.add(row.id);
|
||||
checkComponent(row, monitor)
|
||||
.catch((error) => console.error(`Monitor ${row.id} failed`, error.message))
|
||||
.finally(() => activeChecks.delete(row.id));
|
||||
}
|
||||
}
|
||||
|
||||
function availability(componentId, hours) {
|
||||
const since = new Date(Date.now() - hours * 3600000).toISOString();
|
||||
const result = uptimeSince.get(componentId, since);
|
||||
if (!result || result.total === 0) {
|
||||
return null;
|
||||
}
|
||||
return Math.round((result.successful / result.total) * 10000) / 100;
|
||||
}
|
||||
|
||||
function publicComponent(row, includeAdmin = false) {
|
||||
const metadata = parseJson(row.metadata_json, {id: row.id, name: row.id});
|
||||
const monitor = parseJson(row.monitor_json, null);
|
||||
const incident = openIncident.get(row.id);
|
||||
const component = {
|
||||
...metadata,
|
||||
lifecycle: row.lifecycle,
|
||||
status: row.lifecycle === "active" ? metadata.status : row.lifecycle,
|
||||
monitor: {
|
||||
enabled: Boolean(monitor),
|
||||
status: monitor
|
||||
? row.monitor_paused ? "paused" : row.monitor_state
|
||||
: "unmonitored",
|
||||
checkedAt: row.last_checked_at,
|
||||
responseTimeMs: row.last_latency_ms,
|
||||
statusCode: row.last_status_code,
|
||||
consecutiveFailures: row.consecutive_failures,
|
||||
uptime24h: monitor ? availability(row.id, 24) : null,
|
||||
incidentSince: incident?.started_at || null
|
||||
}
|
||||
};
|
||||
|
||||
if (includeAdmin) {
|
||||
component.firstSeen = row.first_seen;
|
||||
component.lastSeen = row.last_seen;
|
||||
component.missingSince = row.missing_since;
|
||||
component.archivedAt = row.archived_at;
|
||||
component.archiveReason = row.archive_reason;
|
||||
component.monitor.paused = Boolean(row.monitor_paused);
|
||||
component.monitor.error = row.last_error;
|
||||
}
|
||||
|
||||
return component;
|
||||
}
|
||||
|
||||
function componentPayload(includeArchived = false, includeAdmin = false) {
|
||||
const components = selectComponents.all()
|
||||
.filter((row) => includeArchived || row.lifecycle !== "archived")
|
||||
.map((row) => publicComponent(row, includeAdmin))
|
||||
.sort((left, right) => {
|
||||
if (left.section !== right.section) {
|
||||
return left.section.localeCompare(right.section);
|
||||
}
|
||||
if (left.order !== right.order) {
|
||||
return left.order - right.order;
|
||||
}
|
||||
return left.name.localeCompare(right.name, "zh-CN");
|
||||
});
|
||||
|
||||
return {
|
||||
generatedAt: new Date().toISOString(),
|
||||
discoveredAt: lastDiscoveryAt,
|
||||
components
|
||||
};
|
||||
}
|
||||
|
||||
function pruneHistory() {
|
||||
const rawCutoff = new Date(Date.now() - rawRetentionDays * 86400000).toISOString();
|
||||
const aggregateCutoff = new Date(Date.now() - aggregateRetentionDays * 86400000).toISOString();
|
||||
database.prepare("DELETE FROM monitor_check WHERE checked_at < ?").run(rawCutoff);
|
||||
database.prepare("DELETE FROM monitor_hourly WHERE bucket < ?").run(aggregateCutoff);
|
||||
database.prepare("DELETE FROM monitor_daily WHERE bucket < ?").run(aggregateCutoff);
|
||||
}
|
||||
|
||||
function isAdminProxy(request) {
|
||||
return request.headers[adminProxyHeader.toLowerCase()] === "1";
|
||||
}
|
||||
|
||||
function adminAction(id, action) {
|
||||
const row = selectComponent.get(id);
|
||||
if (!row) {
|
||||
return {statusCode: 404, payload: {error: "Component not found"}};
|
||||
}
|
||||
|
||||
const now = new Date().toISOString();
|
||||
if (action === "archive") {
|
||||
database.prepare(`
|
||||
UPDATE registry_component
|
||||
SET lifecycle = 'archived', archived_at = ?, archive_reason = 'manual',
|
||||
monitor_paused = CASE WHEN monitor_json IS NULL THEN 0 ELSE 1 END,
|
||||
monitor_state = CASE WHEN monitor_json IS NULL THEN 'unmonitored' ELSE 'paused' END
|
||||
WHERE id = ?
|
||||
`).run(now, id);
|
||||
} else if (action === "restore") {
|
||||
const recentlySeen = Date.now() - new Date(row.last_seen).getTime() < discoveryIntervalMs * 3;
|
||||
database.prepare(`
|
||||
UPDATE registry_component
|
||||
SET lifecycle = ?, missing_since = ?, archived_at = NULL, archive_reason = NULL,
|
||||
monitor_paused = 0,
|
||||
monitor_state = CASE WHEN monitor_json IS NULL THEN 'unmonitored' ELSE 'pending' END
|
||||
WHERE id = ?
|
||||
`).run(recentlySeen ? "active" : "missing", recentlySeen ? null : now, id);
|
||||
} else if (action === "pause") {
|
||||
if (!row.monitor_json) {
|
||||
return {statusCode: 409, payload: {error: "Monitoring is not configured"}};
|
||||
}
|
||||
database.prepare(`
|
||||
UPDATE registry_component SET monitor_paused = 1, monitor_state = 'paused' WHERE id = ?
|
||||
`).run(id);
|
||||
} else if (action === "resume") {
|
||||
if (!row.monitor_json) {
|
||||
return {statusCode: 409, payload: {error: "Monitoring is not configured"}};
|
||||
}
|
||||
database.prepare(`
|
||||
UPDATE registry_component
|
||||
SET monitor_paused = 0, monitor_state = 'pending', consecutive_failures = 0
|
||||
WHERE id = ?
|
||||
`).run(id);
|
||||
} else if (action === "purge") {
|
||||
const recentlySeen = Date.now() - new Date(row.last_seen).getTime() < discoveryIntervalMs * 3;
|
||||
if (row.lifecycle !== "missing" && recentlySeen) {
|
||||
return {statusCode: 409, payload: {error: "Remove or disable the component Label before purging"}};
|
||||
}
|
||||
database.prepare("DELETE FROM registry_component WHERE id = ?").run(id);
|
||||
return {statusCode: 200, payload: {status: "purged", id}};
|
||||
} else {
|
||||
return {statusCode: 404, payload: {error: "Unknown action"}};
|
||||
}
|
||||
|
||||
return {
|
||||
statusCode: 200,
|
||||
payload: {status: "ok", component: publicComponent(selectComponent.get(id), true)}
|
||||
};
|
||||
}
|
||||
|
||||
function sendJson(response, statusCode, payload, extraHeaders = {}) {
|
||||
const body = JSON.stringify(payload);
|
||||
response.writeHead(statusCode, {
|
||||
"Cache-Control": "no-store",
|
||||
"Content-Type": "application/json; charset=utf-8",
|
||||
"Content-Length": Buffer.byteLength(body),
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
...extraHeaders
|
||||
});
|
||||
response.end(body);
|
||||
}
|
||||
|
||||
const server = http.createServer(async (request, response) => {
|
||||
const pathname = new URL(request.url || "/", "http://localhost").pathname;
|
||||
|
||||
if (request.method === "GET" && pathname === "/healthz") {
|
||||
sendJson(response, 200, {
|
||||
status: "ok",
|
||||
discovery: lastDiscoveryAt ? "ready" : "starting",
|
||||
activeChecks: activeChecks.size
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (request.method === "GET" && pathname === "/components") {
|
||||
sendJson(response, 200, componentPayload(false, false));
|
||||
return;
|
||||
}
|
||||
|
||||
if (pathname.startsWith("/admin/") && !isAdminProxy(request)) {
|
||||
sendJson(response, 403, {error: "Forbidden"});
|
||||
return;
|
||||
}
|
||||
|
||||
if (request.method === "GET" && pathname === "/admin/components") {
|
||||
sendJson(response, 200, componentPayload(true, true));
|
||||
return;
|
||||
}
|
||||
|
||||
const actionMatch = pathname.match(/^\/admin\/components\/([a-z0-9][a-z0-9-]*)\/(archive|restore|pause|resume|purge)$/);
|
||||
if (request.method === "POST" && actionMatch) {
|
||||
const result = adminAction(actionMatch[1], actionMatch[2]);
|
||||
sendJson(response, result.statusCode, result.payload);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!["GET", "POST"].includes(request.method || "")) {
|
||||
sendJson(response, 405, {error: "Method not allowed"}, {"Allow": "GET, POST"});
|
||||
return;
|
||||
}
|
||||
|
||||
sendJson(response, 404, {error: "Not found"});
|
||||
});
|
||||
|
||||
await reconcileDiscovery();
|
||||
pruneHistory();
|
||||
setInterval(reconcileDiscovery, discoveryIntervalMs);
|
||||
setInterval(scheduleDueChecks, 2000);
|
||||
setInterval(pruneHistory, 6 * 3600000);
|
||||
|
||||
server.listen(port, "0.0.0.0", () => {
|
||||
console.log(`component registry and monitor listening on :${port}`);
|
||||
});
|
||||
Reference in New Issue
Block a user