853 lines
27 KiB
JavaScript
853 lines
27 KiB
JavaScript
import http from "node:http";
|
|
import {mkdirSync} from "node:fs";
|
|
import {DatabaseSync} from "node:sqlite";
|
|
|
|
const port = Number.parseInt(process.env.PORT || "8091", 10);
|
|
const dockerSocket = process.env.DOCKER_SOCKET || "/var/run/docker.sock";
|
|
const databasePath = process.env.DATABASE_PATH || "/data/registry.db";
|
|
const discoveryIntervalMs = boundedInteger(process.env.DISCOVERY_INTERVAL_MS, 5000, 2000, 60000);
|
|
const missingRetentionDays = boundedInteger(process.env.MISSING_RETENTION_DAYS, 30, 1, 3650);
|
|
const rawRetentionDays = boundedInteger(process.env.RAW_RETENTION_DAYS, 30, 1, 3650);
|
|
const aggregateRetentionDays = boundedInteger(process.env.AGGREGATE_RETENTION_DAYS, 365, 30, 3650);
|
|
const monitorConcurrency = boundedInteger(process.env.MONITOR_CONCURRENCY, 4, 1, 32);
|
|
const adminProxyHeader = process.env.ADMIN_PROXY_HEADER || "X-Portal-Admin";
|
|
const labelPrefix = "xiteng.site.component.";
|
|
|
|
const allowedFields = new Set([
|
|
"enabled",
|
|
"name",
|
|
"description",
|
|
"section",
|
|
"category",
|
|
"url",
|
|
"endpoint",
|
|
"access",
|
|
"access-label",
|
|
"icon",
|
|
"icon-url",
|
|
"navigation",
|
|
"portal-link",
|
|
"accent",
|
|
"order",
|
|
"monitor.enabled",
|
|
"monitor.url",
|
|
"monitor.interval",
|
|
"monitor.failures",
|
|
"monitor.timeout",
|
|
"monitor.accept"
|
|
]);
|
|
|
|
const allowedSections = new Set(["services", "infrastructure"]);
|
|
const allowedAccents = new Set(["red", "green", "yellow", "blue", "ink"]);
|
|
const activeChecks = new Set();
|
|
let discoveryRunning = false;
|
|
let lastDiscoveryAt = null;
|
|
|
|
mkdirSync(databasePath.slice(0, databasePath.lastIndexOf("/")) || ".", {recursive: true});
|
|
const database = new DatabaseSync(databasePath);
|
|
database.exec(`
|
|
PRAGMA journal_mode = WAL;
|
|
PRAGMA foreign_keys = ON;
|
|
PRAGMA busy_timeout = 5000;
|
|
|
|
CREATE TABLE IF NOT EXISTS registry_component (
|
|
id TEXT PRIMARY KEY,
|
|
first_seen TEXT NOT NULL,
|
|
last_seen TEXT NOT NULL,
|
|
lifecycle TEXT NOT NULL DEFAULT 'active',
|
|
missing_since TEXT,
|
|
archived_at TEXT,
|
|
archive_reason TEXT,
|
|
metadata_json TEXT NOT NULL,
|
|
monitor_json TEXT,
|
|
monitor_state TEXT NOT NULL DEFAULT 'unmonitored',
|
|
monitor_paused INTEGER NOT NULL DEFAULT 0,
|
|
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
|
last_checked_at TEXT,
|
|
last_status_code INTEGER,
|
|
last_latency_ms INTEGER,
|
|
last_error TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS monitor_check (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
component_id TEXT NOT NULL REFERENCES registry_component(id) ON DELETE CASCADE,
|
|
checked_at TEXT NOT NULL,
|
|
ok INTEGER NOT NULL,
|
|
status_code INTEGER,
|
|
latency_ms INTEGER,
|
|
error TEXT
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS monitor_check_component_time
|
|
ON monitor_check(component_id, checked_at);
|
|
|
|
CREATE UNIQUE INDEX IF NOT EXISTS monitor_check_unique
|
|
ON monitor_check(component_id, checked_at);
|
|
|
|
CREATE TABLE IF NOT EXISTS monitor_incident (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
component_id TEXT NOT NULL REFERENCES registry_component(id) ON DELETE CASCADE,
|
|
started_at TEXT NOT NULL,
|
|
ended_at TEXT,
|
|
cause TEXT
|
|
);
|
|
|
|
CREATE UNIQUE INDEX IF NOT EXISTS monitor_incident_open
|
|
ON monitor_incident(component_id) WHERE ended_at IS NULL;
|
|
|
|
CREATE TABLE IF NOT EXISTS monitor_hourly (
|
|
component_id TEXT NOT NULL REFERENCES registry_component(id) ON DELETE CASCADE,
|
|
bucket TEXT NOT NULL,
|
|
total INTEGER NOT NULL DEFAULT 0,
|
|
successful INTEGER NOT NULL DEFAULT 0,
|
|
latency_sum INTEGER NOT NULL DEFAULT 0,
|
|
PRIMARY KEY (component_id, bucket)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS monitor_daily (
|
|
component_id TEXT NOT NULL REFERENCES registry_component(id) ON DELETE CASCADE,
|
|
bucket TEXT NOT NULL,
|
|
total INTEGER NOT NULL DEFAULT 0,
|
|
successful INTEGER NOT NULL DEFAULT 0,
|
|
latency_sum INTEGER NOT NULL DEFAULT 0,
|
|
PRIMARY KEY (component_id, bucket)
|
|
);
|
|
`);
|
|
|
|
const selectComponent = database.prepare("SELECT * FROM registry_component WHERE id = ?");
|
|
const selectComponents = database.prepare("SELECT * FROM registry_component ORDER BY id");
|
|
const insertComponent = database.prepare(`
|
|
INSERT INTO registry_component (
|
|
id, first_seen, last_seen, lifecycle, metadata_json, monitor_json, monitor_state
|
|
) VALUES (?, ?, ?, 'active', ?, ?, ?)
|
|
`);
|
|
const updateDiscoveredComponent = database.prepare(`
|
|
UPDATE registry_component
|
|
SET last_seen = ?,
|
|
lifecycle = ?,
|
|
missing_since = NULL,
|
|
archived_at = CASE WHEN archive_reason = 'manual' THEN archived_at ELSE NULL END,
|
|
archive_reason = CASE WHEN archive_reason = 'manual' THEN archive_reason ELSE NULL END,
|
|
metadata_json = ?,
|
|
monitor_json = ?,
|
|
monitor_state = ?,
|
|
monitor_paused = ?
|
|
WHERE id = ?
|
|
`);
|
|
const markMissing = database.prepare(`
|
|
UPDATE registry_component
|
|
SET lifecycle = 'missing', missing_since = ?
|
|
WHERE id = ? AND lifecycle = 'active'
|
|
`);
|
|
const markArchivedByRetention = database.prepare(`
|
|
UPDATE registry_component
|
|
SET lifecycle = 'archived', archived_at = ?, archive_reason = 'retention',
|
|
monitor_state = CASE WHEN monitor_json IS NULL THEN 'unmonitored' ELSE 'paused' END,
|
|
monitor_paused = CASE WHEN monitor_json IS NULL THEN 0 ELSE 1 END
|
|
WHERE id = ? AND lifecycle = 'missing'
|
|
`);
|
|
const insertCheck = database.prepare(`
|
|
INSERT INTO monitor_check (component_id, checked_at, ok, status_code, latency_ms, error)
|
|
VALUES (?, ?, ?, ?, ?, ?)
|
|
`);
|
|
const upsertHourly = database.prepare(`
|
|
INSERT INTO monitor_hourly (component_id, bucket, total, successful, latency_sum)
|
|
VALUES (?, ?, 1, ?, ?)
|
|
ON CONFLICT(component_id, bucket) DO UPDATE SET
|
|
total = total + 1,
|
|
successful = successful + excluded.successful,
|
|
latency_sum = latency_sum + excluded.latency_sum
|
|
`);
|
|
const upsertDaily = database.prepare(`
|
|
INSERT INTO monitor_daily (component_id, bucket, total, successful, latency_sum)
|
|
VALUES (?, ?, 1, ?, ?)
|
|
ON CONFLICT(component_id, bucket) DO UPDATE SET
|
|
total = total + 1,
|
|
successful = successful + excluded.successful,
|
|
latency_sum = latency_sum + excluded.latency_sum
|
|
`);
|
|
const uptimeSince = database.prepare(`
|
|
SELECT COUNT(*) AS total, COALESCE(SUM(ok), 0) AS successful
|
|
FROM monitor_check
|
|
WHERE component_id = ? AND checked_at >= ?
|
|
`);
|
|
const openIncident = database.prepare(`
|
|
SELECT started_at FROM monitor_incident
|
|
WHERE component_id = ? AND ended_at IS NULL
|
|
`);
|
|
|
|
function boundedInteger(value, fallback, minimum, maximum) {
|
|
const parsed = Number.parseInt(value || "", 10);
|
|
if (!Number.isFinite(parsed)) {
|
|
return fallback;
|
|
}
|
|
return Math.max(minimum, Math.min(maximum, parsed));
|
|
}
|
|
|
|
function dockerGet(pathname) {
|
|
return new Promise((resolve, reject) => {
|
|
const request = http.request({
|
|
socketPath: dockerSocket,
|
|
path: pathname,
|
|
method: "GET",
|
|
headers: {"Accept": "application/json"}
|
|
}, (response) => {
|
|
const chunks = [];
|
|
response.on("data", (chunk) => chunks.push(chunk));
|
|
response.on("end", () => {
|
|
const body = Buffer.concat(chunks).toString("utf8");
|
|
if ((response.statusCode || 500) >= 400) {
|
|
reject(new Error(`Docker API ${response.statusCode}`));
|
|
return;
|
|
}
|
|
|
|
try {
|
|
resolve(JSON.parse(body));
|
|
} catch (error) {
|
|
reject(error);
|
|
}
|
|
});
|
|
});
|
|
|
|
request.setTimeout(3000, () => request.destroy(new Error("Docker API timeout")));
|
|
request.on("error", reject);
|
|
request.end();
|
|
});
|
|
}
|
|
|
|
function limitedString(value, fallback = "", limit = 500) {
|
|
return typeof value === "string" ? value.trim().slice(0, limit) : fallback;
|
|
}
|
|
|
|
function safeUrl(value) {
|
|
const candidate = limitedString(value);
|
|
if (!candidate) {
|
|
return "";
|
|
}
|
|
|
|
try {
|
|
const url = new URL(candidate);
|
|
if (url.username || url.password) {
|
|
return "";
|
|
}
|
|
return ["http:", "https:"].includes(url.protocol) ? url.href : "";
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
function healthFromStatus(status) {
|
|
if (status.includes("(unhealthy)")) {
|
|
return "unhealthy";
|
|
}
|
|
if (status.includes("(healthy)")) {
|
|
return "healthy";
|
|
}
|
|
if (status.includes("(health: starting)")) {
|
|
return "starting";
|
|
}
|
|
return "none";
|
|
}
|
|
|
|
function instanceState(container) {
|
|
const state = limitedString(container.State, "unknown").toLowerCase();
|
|
const health = healthFromStatus(limitedString(container.Status));
|
|
return health === "unhealthy" ? "unhealthy" : state;
|
|
}
|
|
|
|
function aggregateStatus(instances) {
|
|
const states = instances.map((instance) => instance.state);
|
|
if (states.length > 0 && states.every((state) => state === "running")) {
|
|
return "running";
|
|
}
|
|
if (states.includes("unhealthy")) {
|
|
return "unhealthy";
|
|
}
|
|
if (states.includes("restarting")) {
|
|
return "restarting";
|
|
}
|
|
if (states.includes("running")) {
|
|
return "degraded";
|
|
}
|
|
for (const state of ["dead", "exited", "paused", "created"]) {
|
|
if (states.includes(state)) {
|
|
return state;
|
|
}
|
|
}
|
|
return "unknown";
|
|
}
|
|
|
|
function definitionsFromLabels(labels) {
|
|
const definitions = new Map();
|
|
|
|
for (const [key, value] of Object.entries(labels || {})) {
|
|
if (!key.startsWith(labelPrefix)) {
|
|
continue;
|
|
}
|
|
|
|
const remainder = key.slice(labelPrefix.length);
|
|
const separator = remainder.indexOf(".");
|
|
if (separator <= 0) {
|
|
continue;
|
|
}
|
|
|
|
const id = remainder.slice(0, separator);
|
|
const field = remainder.slice(separator + 1);
|
|
if (!/^[a-z0-9][a-z0-9-]*$/.test(id) || !allowedFields.has(field)) {
|
|
continue;
|
|
}
|
|
|
|
if (!definitions.has(id)) {
|
|
definitions.set(id, {});
|
|
}
|
|
definitions.get(id)[field] = limitedString(value);
|
|
}
|
|
|
|
return definitions;
|
|
}
|
|
|
|
function publicDefinition(id, definition) {
|
|
const order = Number.parseInt(definition.order || "999", 10);
|
|
return {
|
|
id,
|
|
name: limitedString(definition.name, id),
|
|
description: limitedString(definition.description),
|
|
section: allowedSections.has(definition.section) ? definition.section : "infrastructure",
|
|
category: limitedString(definition.category, "其他"),
|
|
url: safeUrl(definition.url),
|
|
endpoint: limitedString(definition.endpoint, "", 160),
|
|
access: limitedString(definition.access, "internal", 40),
|
|
accessLabel: limitedString(definition["access-label"], "受控访问", 100),
|
|
icon: limitedString(definition.icon, id.slice(0, 2).toUpperCase(), 8),
|
|
iconUrl: safeUrl(definition["icon-url"]),
|
|
navigation: ["new-tab", "same-tab", "endpoint"].includes(definition.navigation) ? definition.navigation : (definition.url ? "new-tab" : "endpoint"),
|
|
portalLink: ["embedded", "native", "none"].includes(definition["portal-link"]) ? definition["portal-link"] : "none",
|
|
accent: allowedAccents.has(definition.accent) ? definition.accent : "ink",
|
|
order: Number.isFinite(order) ? order : 999
|
|
};
|
|
}
|
|
|
|
function monitorDefinition(definition) {
|
|
if (definition["monitor.enabled"]?.toLowerCase() !== "true") {
|
|
return null;
|
|
}
|
|
|
|
const url = safeUrl(definition["monitor.url"]);
|
|
if (!url) {
|
|
return null;
|
|
}
|
|
|
|
return {
|
|
type: "http",
|
|
url,
|
|
interval: boundedInteger(definition["monitor.interval"], 60, 10, 86400),
|
|
failures: boundedInteger(definition["monitor.failures"], 3, 1, 20),
|
|
timeout: boundedInteger(definition["monitor.timeout"], 10, 1, 60),
|
|
accept: limitedString(definition["monitor.accept"], "200-299", 80)
|
|
};
|
|
}
|
|
|
|
function parseJson(value, fallback) {
|
|
try {
|
|
return value ? JSON.parse(value) : fallback;
|
|
} catch {
|
|
return fallback;
|
|
}
|
|
}
|
|
|
|
async function discoverDockerComponents() {
|
|
const containers = await dockerGet("/containers/json?all=true");
|
|
const components = new Map();
|
|
|
|
for (const container of containers) {
|
|
const labels = container.Labels || {};
|
|
for (const [id, definition] of definitionsFromLabels(labels)) {
|
|
if (definition.enabled?.toLowerCase() !== "true") {
|
|
continue;
|
|
}
|
|
|
|
const metadata = publicDefinition(id, definition);
|
|
const instance = {
|
|
container: limitedString(container.Names?.[0] || "", "", 160).replace(/^\//, ""),
|
|
service: limitedString(labels["com.docker.compose.service"], "docker", 160),
|
|
project: limitedString(labels["com.docker.compose.project"], "", 160),
|
|
image: limitedString(container.Image, "", 300),
|
|
state: instanceState(container),
|
|
health: healthFromStatus(limitedString(container.Status))
|
|
};
|
|
|
|
if (!components.has(id)) {
|
|
components.set(id, {...metadata, monitor: monitorDefinition(definition), instances: []});
|
|
}
|
|
components.get(id).instances.push(instance);
|
|
}
|
|
}
|
|
|
|
for (const component of components.values()) {
|
|
const primary = component.instances[0];
|
|
component.status = aggregateStatus(component.instances);
|
|
component.instanceCount = component.instances.length;
|
|
component.container = primary.container;
|
|
component.service = primary.service;
|
|
component.project = primary.project;
|
|
component.image = primary.image;
|
|
}
|
|
|
|
return components;
|
|
}
|
|
|
|
function reconcileComponent(id, discovered, now) {
|
|
const existing = selectComponent.get(id);
|
|
const metadata = {...discovered};
|
|
delete metadata.monitor;
|
|
delete metadata.instances;
|
|
const metadataJson = JSON.stringify(metadata);
|
|
const monitorJson = discovered.monitor ? JSON.stringify(discovered.monitor) : null;
|
|
|
|
if (!existing) {
|
|
insertComponent.run(
|
|
id,
|
|
now,
|
|
now,
|
|
metadataJson,
|
|
monitorJson,
|
|
monitorJson ? "pending" : "unmonitored"
|
|
);
|
|
return;
|
|
}
|
|
|
|
const manuallyArchived = existing.lifecycle === "archived" && existing.archive_reason === "manual";
|
|
const monitorChanged = existing.monitor_json !== monitorJson;
|
|
let monitorState = existing.monitor_state;
|
|
let monitorPaused = existing.monitor_paused;
|
|
|
|
if (!monitorJson) {
|
|
monitorState = "unmonitored";
|
|
monitorPaused = 0;
|
|
} else if (manuallyArchived) {
|
|
monitorState = "paused";
|
|
monitorPaused = 1;
|
|
} else if (monitorChanged || existing.lifecycle === "archived") {
|
|
monitorState = "pending";
|
|
monitorPaused = 0;
|
|
}
|
|
|
|
updateDiscoveredComponent.run(
|
|
now,
|
|
manuallyArchived ? "archived" : "active",
|
|
metadataJson,
|
|
monitorJson,
|
|
monitorState,
|
|
monitorPaused,
|
|
id
|
|
);
|
|
|
|
if (monitorChanged) {
|
|
database.prepare(`
|
|
UPDATE registry_component
|
|
SET consecutive_failures = 0, last_checked_at = NULL, last_status_code = NULL,
|
|
last_latency_ms = NULL, last_error = NULL
|
|
WHERE id = ?
|
|
`).run(id);
|
|
database.prepare(`
|
|
UPDATE monitor_incident SET ended_at = ?
|
|
WHERE component_id = ? AND ended_at IS NULL
|
|
`).run(now, id);
|
|
}
|
|
}
|
|
|
|
async function reconcileDiscovery() {
|
|
if (discoveryRunning) {
|
|
return;
|
|
}
|
|
discoveryRunning = true;
|
|
|
|
try {
|
|
const now = new Date().toISOString();
|
|
const discovered = await discoverDockerComponents();
|
|
|
|
database.exec("BEGIN IMMEDIATE");
|
|
try {
|
|
for (const [id, component] of discovered) {
|
|
reconcileComponent(id, component, now);
|
|
}
|
|
|
|
for (const existing of selectComponents.all()) {
|
|
if (discovered.has(existing.id) || existing.lifecycle === "archived") {
|
|
continue;
|
|
}
|
|
|
|
if (existing.lifecycle === "active") {
|
|
markMissing.run(now, existing.id);
|
|
continue;
|
|
}
|
|
|
|
if (existing.lifecycle === "missing" && existing.missing_since) {
|
|
const missingForMs = Date.now() - new Date(existing.missing_since).getTime();
|
|
if (missingForMs >= missingRetentionDays * 86400000) {
|
|
markArchivedByRetention.run(now, existing.id);
|
|
}
|
|
}
|
|
}
|
|
database.exec("COMMIT");
|
|
} catch (error) {
|
|
database.exec("ROLLBACK");
|
|
throw error;
|
|
}
|
|
|
|
lastDiscoveryAt = now;
|
|
} catch (error) {
|
|
console.error("Docker discovery failed", error.message);
|
|
} finally {
|
|
discoveryRunning = false;
|
|
}
|
|
}
|
|
|
|
function statusAccepted(statusCode, expression) {
|
|
for (const part of expression.split(",")) {
|
|
const match = part.trim().match(/^(\d{3})(?:-(\d{3}))?$/);
|
|
if (!match) {
|
|
continue;
|
|
}
|
|
const minimum = Number.parseInt(match[1], 10);
|
|
const maximum = Number.parseInt(match[2] || match[1], 10);
|
|
if (statusCode >= minimum && statusCode <= maximum) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
function safeError(error) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
return limitedString(message.replace(/[\r\n\t]+/g, " "), "probe failed", 300);
|
|
}
|
|
|
|
function updateAggregate(componentId, checkedAt, ok, latencyMs) {
|
|
const hourBucket = `${checkedAt.slice(0, 13)}:00:00.000Z`;
|
|
const dayBucket = `${checkedAt.slice(0, 10)}T00:00:00.000Z`;
|
|
upsertHourly.run(componentId, hourBucket, ok ? 1 : 0, latencyMs || 0);
|
|
upsertDaily.run(componentId, dayBucket, ok ? 1 : 0, latencyMs || 0);
|
|
}
|
|
|
|
function recordCheck(componentId, monitor, result) {
|
|
const existing = selectComponent.get(componentId);
|
|
if (!existing) {
|
|
return;
|
|
}
|
|
|
|
const nextFailures = result.ok ? 0 : existing.consecutive_failures + 1;
|
|
const nextState = result.ok ? "up" : nextFailures >= monitor.failures ? "down" : "degraded";
|
|
|
|
database.exec("BEGIN IMMEDIATE");
|
|
try {
|
|
insertCheck.run(
|
|
componentId,
|
|
result.checkedAt,
|
|
result.ok ? 1 : 0,
|
|
result.statusCode,
|
|
result.latencyMs,
|
|
result.error
|
|
);
|
|
updateAggregate(componentId, result.checkedAt, result.ok, result.latencyMs);
|
|
|
|
database.prepare(`
|
|
UPDATE registry_component
|
|
SET monitor_state = ?, consecutive_failures = ?, last_checked_at = ?,
|
|
last_status_code = ?, last_latency_ms = ?, last_error = ?
|
|
WHERE id = ?
|
|
`).run(
|
|
nextState,
|
|
nextFailures,
|
|
result.checkedAt,
|
|
result.statusCode,
|
|
result.latencyMs,
|
|
result.error,
|
|
componentId
|
|
);
|
|
|
|
if (nextState === "down" && existing.monitor_state !== "down") {
|
|
database.prepare(`
|
|
INSERT OR IGNORE INTO monitor_incident (component_id, started_at, cause)
|
|
VALUES (?, ?, ?)
|
|
`).run(componentId, result.checkedAt, result.error || `HTTP ${result.statusCode || "error"}`);
|
|
}
|
|
|
|
if (result.ok) {
|
|
database.prepare(`
|
|
UPDATE monitor_incident SET ended_at = ?
|
|
WHERE component_id = ? AND ended_at IS NULL
|
|
`).run(result.checkedAt, componentId);
|
|
}
|
|
database.exec("COMMIT");
|
|
} catch (error) {
|
|
database.exec("ROLLBACK");
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function checkComponent(row, monitor) {
|
|
const checkedAt = new Date().toISOString();
|
|
const startedAt = performance.now();
|
|
let statusCode = null;
|
|
let ok = false;
|
|
let errorMessage = null;
|
|
|
|
try {
|
|
const response = await fetch(monitor.url, {
|
|
method: "GET",
|
|
redirect: "follow",
|
|
headers: {
|
|
"Accept": "text/html,application/json;q=0.9,*/*;q=0.1",
|
|
"User-Agent": "Xiteng-Portal-Monitor/1.0"
|
|
},
|
|
signal: AbortSignal.timeout(monitor.timeout * 1000)
|
|
});
|
|
statusCode = response.status;
|
|
ok = statusAccepted(statusCode, monitor.accept);
|
|
if (!ok) {
|
|
errorMessage = `HTTP ${statusCode}`;
|
|
}
|
|
if (response.body) {
|
|
await response.body.cancel();
|
|
}
|
|
} catch (error) {
|
|
errorMessage = safeError(error);
|
|
}
|
|
|
|
const latencyMs = Math.max(0, Math.round(performance.now() - startedAt));
|
|
recordCheck(row.id, monitor, {
|
|
checkedAt,
|
|
ok,
|
|
statusCode,
|
|
latencyMs,
|
|
error: errorMessage
|
|
});
|
|
}
|
|
|
|
function scheduleDueChecks() {
|
|
if (activeChecks.size >= monitorConcurrency) {
|
|
return;
|
|
}
|
|
|
|
const now = Date.now();
|
|
const candidates = selectComponents.all().filter((row) => {
|
|
if (!row.monitor_json || row.monitor_paused || !["active", "missing"].includes(row.lifecycle)) {
|
|
return false;
|
|
}
|
|
const monitor = parseJson(row.monitor_json, null);
|
|
if (!monitor) {
|
|
return false;
|
|
}
|
|
const lastChecked = row.last_checked_at ? new Date(row.last_checked_at).getTime() : 0;
|
|
return !activeChecks.has(row.id) && now - lastChecked >= monitor.interval * 1000;
|
|
});
|
|
|
|
for (const row of candidates.slice(0, monitorConcurrency - activeChecks.size)) {
|
|
const monitor = parseJson(row.monitor_json, null);
|
|
activeChecks.add(row.id);
|
|
checkComponent(row, monitor)
|
|
.catch((error) => console.error(`Monitor ${row.id} failed`, error.message))
|
|
.finally(() => activeChecks.delete(row.id));
|
|
}
|
|
}
|
|
|
|
function availability(componentId, hours) {
|
|
const since = new Date(Date.now() - hours * 3600000).toISOString();
|
|
const result = uptimeSince.get(componentId, since);
|
|
if (!result || result.total === 0) {
|
|
return null;
|
|
}
|
|
return Math.round((result.successful / result.total) * 10000) / 100;
|
|
}
|
|
|
|
function publicComponent(row, includeAdmin = false) {
|
|
const metadata = parseJson(row.metadata_json, {id: row.id, name: row.id});
|
|
const monitor = parseJson(row.monitor_json, null);
|
|
const incident = openIncident.get(row.id);
|
|
const component = {
|
|
...metadata,
|
|
lifecycle: row.lifecycle,
|
|
status: row.lifecycle === "active" ? metadata.status : row.lifecycle,
|
|
monitor: {
|
|
enabled: Boolean(monitor),
|
|
status: monitor
|
|
? row.monitor_paused ? "paused" : row.monitor_state
|
|
: "unmonitored",
|
|
checkedAt: row.last_checked_at,
|
|
responseTimeMs: row.last_latency_ms,
|
|
statusCode: row.last_status_code,
|
|
consecutiveFailures: row.consecutive_failures,
|
|
uptime24h: monitor ? availability(row.id, 24) : null,
|
|
incidentSince: incident?.started_at || null
|
|
}
|
|
};
|
|
|
|
if (includeAdmin) {
|
|
component.firstSeen = row.first_seen;
|
|
component.lastSeen = row.last_seen;
|
|
component.missingSince = row.missing_since;
|
|
component.archivedAt = row.archived_at;
|
|
component.archiveReason = row.archive_reason;
|
|
component.monitor.paused = Boolean(row.monitor_paused);
|
|
component.monitor.error = row.last_error;
|
|
}
|
|
|
|
return component;
|
|
}
|
|
|
|
function componentPayload(includeArchived = false, includeAdmin = false) {
|
|
const components = selectComponents.all()
|
|
.filter((row) => includeArchived || row.lifecycle !== "archived")
|
|
.map((row) => publicComponent(row, includeAdmin))
|
|
.sort((left, right) => {
|
|
if (left.section !== right.section) {
|
|
return left.section.localeCompare(right.section);
|
|
}
|
|
if (left.order !== right.order) {
|
|
return left.order - right.order;
|
|
}
|
|
return left.name.localeCompare(right.name, "zh-CN");
|
|
});
|
|
|
|
return {
|
|
generatedAt: new Date().toISOString(),
|
|
discoveredAt: lastDiscoveryAt,
|
|
components
|
|
};
|
|
}
|
|
|
|
function pruneHistory() {
|
|
const rawCutoff = new Date(Date.now() - rawRetentionDays * 86400000).toISOString();
|
|
const aggregateCutoff = new Date(Date.now() - aggregateRetentionDays * 86400000).toISOString();
|
|
database.prepare("DELETE FROM monitor_check WHERE checked_at < ?").run(rawCutoff);
|
|
database.prepare("DELETE FROM monitor_hourly WHERE bucket < ?").run(aggregateCutoff);
|
|
database.prepare("DELETE FROM monitor_daily WHERE bucket < ?").run(aggregateCutoff);
|
|
}
|
|
|
|
function isAdminProxy(request) {
|
|
return request.headers[adminProxyHeader.toLowerCase()] === "1";
|
|
}
|
|
|
|
function adminAction(id, action) {
|
|
const row = selectComponent.get(id);
|
|
if (!row) {
|
|
return {statusCode: 404, payload: {error: "Component not found"}};
|
|
}
|
|
|
|
const now = new Date().toISOString();
|
|
if (action === "archive") {
|
|
database.prepare(`
|
|
UPDATE registry_component
|
|
SET lifecycle = 'archived', archived_at = ?, archive_reason = 'manual',
|
|
monitor_paused = CASE WHEN monitor_json IS NULL THEN 0 ELSE 1 END,
|
|
monitor_state = CASE WHEN monitor_json IS NULL THEN 'unmonitored' ELSE 'paused' END
|
|
WHERE id = ?
|
|
`).run(now, id);
|
|
} else if (action === "restore") {
|
|
const recentlySeen = Date.now() - new Date(row.last_seen).getTime() < discoveryIntervalMs * 3;
|
|
database.prepare(`
|
|
UPDATE registry_component
|
|
SET lifecycle = ?, missing_since = ?, archived_at = NULL, archive_reason = NULL,
|
|
monitor_paused = 0,
|
|
monitor_state = CASE WHEN monitor_json IS NULL THEN 'unmonitored' ELSE 'pending' END
|
|
WHERE id = ?
|
|
`).run(recentlySeen ? "active" : "missing", recentlySeen ? null : now, id);
|
|
} else if (action === "pause") {
|
|
if (!row.monitor_json) {
|
|
return {statusCode: 409, payload: {error: "Monitoring is not configured"}};
|
|
}
|
|
database.prepare(`
|
|
UPDATE registry_component SET monitor_paused = 1, monitor_state = 'paused' WHERE id = ?
|
|
`).run(id);
|
|
} else if (action === "resume") {
|
|
if (!row.monitor_json) {
|
|
return {statusCode: 409, payload: {error: "Monitoring is not configured"}};
|
|
}
|
|
database.prepare(`
|
|
UPDATE registry_component
|
|
SET monitor_paused = 0, monitor_state = 'pending', consecutive_failures = 0
|
|
WHERE id = ?
|
|
`).run(id);
|
|
} else if (action === "purge") {
|
|
const recentlySeen = Date.now() - new Date(row.last_seen).getTime() < discoveryIntervalMs * 3;
|
|
if (row.lifecycle !== "missing" && recentlySeen) {
|
|
return {statusCode: 409, payload: {error: "Remove or disable the component Label before purging"}};
|
|
}
|
|
database.prepare("DELETE FROM registry_component WHERE id = ?").run(id);
|
|
return {statusCode: 200, payload: {status: "purged", id}};
|
|
} else {
|
|
return {statusCode: 404, payload: {error: "Unknown action"}};
|
|
}
|
|
|
|
return {
|
|
statusCode: 200,
|
|
payload: {status: "ok", component: publicComponent(selectComponent.get(id), true)}
|
|
};
|
|
}
|
|
|
|
function sendJson(response, statusCode, payload, extraHeaders = {}) {
|
|
const body = JSON.stringify(payload);
|
|
response.writeHead(statusCode, {
|
|
"Cache-Control": "no-store",
|
|
"Content-Type": "application/json; charset=utf-8",
|
|
"Content-Length": Buffer.byteLength(body),
|
|
"X-Content-Type-Options": "nosniff",
|
|
...extraHeaders
|
|
});
|
|
response.end(body);
|
|
}
|
|
|
|
const server = http.createServer(async (request, response) => {
|
|
const pathname = new URL(request.url || "/", "http://localhost").pathname;
|
|
|
|
if (request.method === "GET" && pathname === "/healthz") {
|
|
sendJson(response, 200, {
|
|
status: "ok",
|
|
discovery: lastDiscoveryAt ? "ready" : "starting",
|
|
activeChecks: activeChecks.size
|
|
});
|
|
return;
|
|
}
|
|
|
|
if (request.method === "GET" && pathname === "/components") {
|
|
sendJson(response, 200, componentPayload(false, false));
|
|
return;
|
|
}
|
|
|
|
if (pathname.startsWith("/admin/") && !isAdminProxy(request)) {
|
|
sendJson(response, 403, {error: "Forbidden"});
|
|
return;
|
|
}
|
|
|
|
if (request.method === "GET" && pathname === "/admin/components") {
|
|
sendJson(response, 200, componentPayload(true, true));
|
|
return;
|
|
}
|
|
|
|
const actionMatch = pathname.match(/^\/admin\/components\/([a-z0-9][a-z0-9-]*)\/(archive|restore|pause|resume|purge)$/);
|
|
if (request.method === "POST" && actionMatch) {
|
|
const result = adminAction(actionMatch[1], actionMatch[2]);
|
|
sendJson(response, result.statusCode, result.payload);
|
|
return;
|
|
}
|
|
|
|
if (!["GET", "POST"].includes(request.method || "")) {
|
|
sendJson(response, 405, {error: "Method not allowed"}, {"Allow": "GET, POST"});
|
|
return;
|
|
}
|
|
|
|
sendJson(response, 404, {error: "Not found"});
|
|
});
|
|
|
|
await reconcileDiscovery();
|
|
pruneHistory();
|
|
setInterval(reconcileDiscovery, discoveryIntervalMs);
|
|
setInterval(scheduleDueChecks, 2000);
|
|
setInterval(pruneHistory, 6 * 3600000);
|
|
|
|
server.listen(port, "0.0.0.0", () => {
|
|
console.log(`component registry and monitor listening on :${port}`);
|
|
});
|