feat: rebuild xiteng.site homelab platform
This commit is contained in:
@@ -0,0 +1,509 @@
|
||||
import http from "node:http";
|
||||
import {createHash} from "node:crypto";
|
||||
import {mkdirSync, readFileSync, renameSync, writeFileSync} from "node:fs";
|
||||
|
||||
const labelPrefix = "xiteng.site.cache.";
|
||||
const allowedFields = new Set([
|
||||
"enabled",
|
||||
"routers",
|
||||
"hosts",
|
||||
"paths",
|
||||
"edge-ttl",
|
||||
"browser-ttl",
|
||||
"stale-while-revalidate"
|
||||
]);
|
||||
const managedRuleDescription = "Xiteng label-managed static paths";
|
||||
|
||||
function boundedInteger(value, fallback, minimum, maximum) {
|
||||
const parsed = Number.parseInt(value || "", 10);
|
||||
return Number.isFinite(parsed) ? Math.max(minimum, Math.min(maximum, parsed)) : fallback;
|
||||
}
|
||||
|
||||
function csv(value) {
|
||||
return [...new Set(String(value || "").split(",").map((item) => item.trim()).filter(Boolean))];
|
||||
}
|
||||
|
||||
function stableName(...parts) {
|
||||
const source = parts.join("-").toLowerCase();
|
||||
const readable = source.replace(/[^a-z0-9-]+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "").slice(0, 48);
|
||||
const digest = createHash("sha256").update(source).digest("hex").slice(0, 10);
|
||||
return `${readable || "cache"}-${digest}`;
|
||||
}
|
||||
|
||||
function normalizedLabels(labels) {
|
||||
return Object.fromEntries(Object.entries(labels || {}).map(([key, value]) => [key.toLowerCase(), String(value)]));
|
||||
}
|
||||
|
||||
function normalizePathPrefix(value) {
|
||||
const path = String(value || "").trim();
|
||||
if (!path.startsWith("/") || !path.endsWith("/") || path === "/") {
|
||||
throw new Error(`static path must start and end with / and cannot be root: ${path || "(empty)"}`);
|
||||
}
|
||||
if (path.includes("..") || path.includes("//") || !/^\/[A-Za-z0-9._~/-]+\/$/.test(path)) {
|
||||
throw new Error(`static path contains unsafe characters: ${path}`);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
function normalizeHost(value, zoneName = "") {
|
||||
const host = String(value || "").trim().toLowerCase().replace(/\.$/, "");
|
||||
if (!/^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$/.test(host)) {
|
||||
throw new Error(`invalid hostname: ${value}`);
|
||||
}
|
||||
if (zoneName && host !== zoneName && !host.endsWith(`.${zoneName}`)) {
|
||||
throw new Error(`hostname is outside ${zoneName}: ${host}`);
|
||||
}
|
||||
return host;
|
||||
}
|
||||
|
||||
function extractHosts(rule, zoneName) {
|
||||
const hosts = [];
|
||||
for (const match of String(rule || "").matchAll(/\bHost\(([^)]*)\)/g)) {
|
||||
for (const quoted of match[1].matchAll(/`([^`]+)`/g)) {
|
||||
hosts.push(normalizeHost(quoted[1], zoneName));
|
||||
}
|
||||
}
|
||||
return [...new Set(hosts)];
|
||||
}
|
||||
|
||||
function qualifyReference(value, provider) {
|
||||
const reference = String(value || "").trim();
|
||||
return !reference || reference.includes("@") ? reference : `${reference}@${provider}`;
|
||||
}
|
||||
|
||||
export function definitionsFromLabels(rawLabels) {
|
||||
const labels = normalizedLabels(rawLabels);
|
||||
const definitions = new Map();
|
||||
for (const [key, value] of Object.entries(labels)) {
|
||||
if (!key.startsWith(labelPrefix)) {
|
||||
continue;
|
||||
}
|
||||
const remainder = key.slice(labelPrefix.length);
|
||||
const separator = remainder.indexOf(".");
|
||||
const id = remainder.slice(0, separator);
|
||||
const field = remainder.slice(separator + 1);
|
||||
if (separator <= 0 || !/^[a-z0-9][a-z0-9-]*$/.test(id) || !allowedFields.has(field)) {
|
||||
continue;
|
||||
}
|
||||
if (!definitions.has(id)) {
|
||||
definitions.set(id, {});
|
||||
}
|
||||
definitions.get(id)[field] = value.trim();
|
||||
}
|
||||
return definitions;
|
||||
}
|
||||
|
||||
function routerNames(labels) {
|
||||
const names = new Set();
|
||||
for (const key of Object.keys(labels)) {
|
||||
const match = key.match(/^traefik\.http\.routers\.([a-z0-9-]+)\.rule$/);
|
||||
if (match) {
|
||||
names.add(match[1]);
|
||||
}
|
||||
}
|
||||
return [...names].sort();
|
||||
}
|
||||
|
||||
function serviceNames(labels) {
|
||||
const names = new Set();
|
||||
for (const key of Object.keys(labels)) {
|
||||
const match = key.match(/^traefik\.http\.services\.([a-z0-9-]+)\.loadbalancer\.server\.port$/);
|
||||
if (match) {
|
||||
names.add(match[1]);
|
||||
}
|
||||
}
|
||||
return [...names];
|
||||
}
|
||||
|
||||
function cloudflareExpression(policies) {
|
||||
const clauses = [];
|
||||
for (const policy of policies) {
|
||||
const hosts = policy.hosts.map((host) => `http.host eq ${JSON.stringify(host)}`);
|
||||
const paths = policy.paths.map((path) => `starts_with(http.request.uri.path, ${JSON.stringify(path)})`);
|
||||
clauses.push(`((${hosts.join(" or ")}) and (${paths.join(" or ")}))`);
|
||||
}
|
||||
return clauses.length ? `(${clauses.join(" or ")})` : "(http.host eq \"cache-disabled.invalid\")";
|
||||
}
|
||||
|
||||
export function buildConfiguration(containers, {zoneName = "xiteng.site"} = {}) {
|
||||
const routers = {};
|
||||
const middlewares = {};
|
||||
const cloudflarePolicies = [];
|
||||
const errors = [];
|
||||
const seenPolicies = new Set();
|
||||
|
||||
for (const container of containers || []) {
|
||||
const labels = normalizedLabels(container.Labels);
|
||||
if (labels["traefik.enable"] !== "true") {
|
||||
continue;
|
||||
}
|
||||
const project = labels["com.docker.compose.project"] || "docker";
|
||||
const service = labels["com.docker.compose.service"] || String(container.Names?.[0] || container.Id || "service").replace(/^\//, "");
|
||||
const definitions = definitionsFromLabels(labels);
|
||||
|
||||
for (const [policyId, definition] of definitions) {
|
||||
if (definition.enabled !== "true") {
|
||||
continue;
|
||||
}
|
||||
const policyKey = `${project}/${service}/${policyId}`;
|
||||
if (seenPolicies.has(policyKey)) {
|
||||
continue;
|
||||
}
|
||||
seenPolicies.add(policyKey);
|
||||
|
||||
try {
|
||||
const paths = csv(definition.paths).map(normalizePathPrefix).sort();
|
||||
if (!paths.length) {
|
||||
throw new Error("at least one static path is required");
|
||||
}
|
||||
const selectedRouters = (csv(definition.routers).length ? csv(definition.routers) : routerNames(labels)).sort();
|
||||
if (!selectedRouters.length) {
|
||||
throw new Error("no Traefik routers found; set .routers explicitly");
|
||||
}
|
||||
const edgeTtl = boundedInteger(definition["edge-ttl"], 604800, 60, 31536000);
|
||||
const browserTtl = boundedInteger(definition["browser-ttl"], 0, 0, 31536000);
|
||||
const staleWhileRevalidate = boundedInteger(definition["stale-while-revalidate"], 86400, 0, 604800);
|
||||
const middlewareName = `xiteng-static-${stableName(policyKey)}`;
|
||||
const responseHeaders = {
|
||||
"Cloudflare-CDN-Cache-Control": `public, s-maxage=${edgeTtl}${staleWhileRevalidate ? `, stale-while-revalidate=${staleWhileRevalidate}` : ""}`,
|
||||
"X-Xiteng-Cache-Policy": policyKey
|
||||
};
|
||||
if (browserTtl > 0) {
|
||||
responseHeaders["Cache-Control"] = `public, max-age=${browserTtl}${staleWhileRevalidate ? `, stale-while-revalidate=${staleWhileRevalidate}` : ""}`;
|
||||
}
|
||||
const policyRouters = {};
|
||||
const discoveredHosts = new Set(csv(definition.hosts).map((host) => normalizeHost(host, zoneName)));
|
||||
for (const baseRouter of selectedRouters) {
|
||||
if (!/^[a-z0-9][a-z0-9-]*$/.test(baseRouter)) {
|
||||
throw new Error(`invalid router name: ${baseRouter}`);
|
||||
}
|
||||
const routerPrefix = `traefik.http.routers.${baseRouter}.`;
|
||||
const baseRule = labels[`${routerPrefix}rule`];
|
||||
if (!baseRule) {
|
||||
throw new Error(`router ${baseRouter} has no rule on this service`);
|
||||
}
|
||||
for (const host of extractHosts(baseRule, zoneName)) {
|
||||
discoveredHosts.add(host);
|
||||
}
|
||||
const entryPoints = csv(labels[`${routerPrefix}entrypoints`]);
|
||||
if (!entryPoints.length) {
|
||||
throw new Error(`router ${baseRouter} must declare entrypoints`);
|
||||
}
|
||||
let targetService = labels[`${routerPrefix}service`];
|
||||
if (!targetService) {
|
||||
const candidates = serviceNames(labels);
|
||||
if (candidates.length !== 1) {
|
||||
throw new Error(`router ${baseRouter} must declare its Traefik service`);
|
||||
}
|
||||
[targetService] = candidates;
|
||||
}
|
||||
const inheritedMiddlewares = csv(labels[`${routerPrefix}middlewares`]).map((item) => qualifyReference(item, "docker"));
|
||||
const generatedRouter = {
|
||||
rule: `(${baseRule}) && (Method(\`GET\`) || Method(\`HEAD\`)) && (${paths.map((path) => `PathPrefix(\`${path}\`)`).join(" || ")})`,
|
||||
entryPoints,
|
||||
service: qualifyReference(targetService, "docker"),
|
||||
// Traefik runs response middleware in reverse order. Put the cache
|
||||
// middleware first so it is the final writer of cache headers.
|
||||
middlewares: [middlewareName, ...inheritedMiddlewares],
|
||||
priority: boundedInteger(labels[`${routerPrefix}priority`], 0, 0, 1000000) + 1000
|
||||
};
|
||||
if (labels[`${routerPrefix}tls`] && labels[`${routerPrefix}tls`] !== "false") {
|
||||
generatedRouter.tls = {};
|
||||
if (labels[`${routerPrefix}tls.certresolver`]) {
|
||||
generatedRouter.tls.certResolver = labels[`${routerPrefix}tls.certresolver`];
|
||||
}
|
||||
if (labels[`${routerPrefix}tls.options`]) {
|
||||
generatedRouter.tls.options = qualifyReference(labels[`${routerPrefix}tls.options`], "docker");
|
||||
}
|
||||
}
|
||||
policyRouters[`xiteng-static-${stableName(policyKey, baseRouter)}`] = generatedRouter;
|
||||
}
|
||||
if (!discoveredHosts.size) {
|
||||
throw new Error("no exact Host() value found; set .hosts explicitly");
|
||||
}
|
||||
middlewares[middlewareName] = {headers: {customResponseHeaders: responseHeaders}};
|
||||
Object.assign(routers, policyRouters);
|
||||
cloudflarePolicies.push({
|
||||
id: policyKey,
|
||||
hosts: [...discoveredHosts].sort(),
|
||||
paths
|
||||
});
|
||||
} catch (error) {
|
||||
errors.push(`${policyKey}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cloudflarePolicies.sort((left, right) => left.id.localeCompare(right.id));
|
||||
return {
|
||||
traefik: {http: {routers, middlewares}},
|
||||
cloudflare: {
|
||||
version: 1,
|
||||
zone: zoneName,
|
||||
description: managedRuleDescription,
|
||||
expression: cloudflareExpression(cloudflarePolicies),
|
||||
policies: cloudflarePolicies
|
||||
},
|
||||
errors,
|
||||
policyCount: cloudflarePolicies.length,
|
||||
routerCount: Object.keys(routers).length
|
||||
};
|
||||
}
|
||||
|
||||
export function cloudflareRuleFromSpec(spec) {
|
||||
return {
|
||||
action: "set_cache_settings",
|
||||
action_parameters: {
|
||||
cache: true,
|
||||
edge_ttl: {
|
||||
mode: "respect_origin",
|
||||
status_code_ttl: [
|
||||
{status_code_range: {to: 199}, value: 0},
|
||||
{status_code_range: {from: 300, to: 499}, value: 0},
|
||||
{status_code_range: {from: 500}, value: -1}
|
||||
]
|
||||
},
|
||||
browser_ttl: {mode: "respect_origin"},
|
||||
cache_key: {cache_deception_armor: true},
|
||||
serve_stale: {disable_stale_while_updating: false}
|
||||
},
|
||||
expression: spec.expression,
|
||||
description: managedRuleDescription,
|
||||
enabled: Array.isArray(spec.policies) && spec.policies.length > 0
|
||||
};
|
||||
}
|
||||
|
||||
async function cloudflareRequest(path, {token, method = "GET", body} = {}) {
|
||||
const response = await fetch(`https://api.cloudflare.com/client/v4${path}`, {
|
||||
method,
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
signal: AbortSignal.timeout(15000)
|
||||
});
|
||||
const payload = await response.json();
|
||||
if (!response.ok || !payload.success) {
|
||||
const detail = (payload.errors || []).map((error) => `${error.code}: ${error.message}`).join("; ");
|
||||
const error = new Error(`Cloudflare API ${response.status}${detail ? ` (${detail})` : ""}`);
|
||||
error.status = response.status;
|
||||
throw error;
|
||||
}
|
||||
return payload.result;
|
||||
}
|
||||
|
||||
function canonicalizeJson(value) {
|
||||
if (Array.isArray(value)) {
|
||||
return value.map(canonicalizeJson);
|
||||
}
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(
|
||||
Object.keys(value).sort().map((key) => [key, canonicalizeJson(value[key])])
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function comparableRule(rule) {
|
||||
return JSON.stringify(canonicalizeJson({
|
||||
action: rule.action,
|
||||
action_parameters: rule.action_parameters,
|
||||
expression: rule.expression,
|
||||
description: rule.description,
|
||||
enabled: rule.enabled !== false
|
||||
}));
|
||||
}
|
||||
|
||||
export async function syncCloudflareRule(spec, {zoneId, token, dryRun = false} = {}) {
|
||||
if (!zoneId || !token) {
|
||||
throw new Error("Cloudflare zone ID and API token are required");
|
||||
}
|
||||
const desired = cloudflareRuleFromSpec(spec);
|
||||
let ruleset;
|
||||
try {
|
||||
ruleset = await cloudflareRequest(`/zones/${zoneId}/rulesets/phases/http_request_cache_settings/entrypoint`, {token});
|
||||
} catch (error) {
|
||||
if (error.status !== 404) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
const current = ruleset?.rules?.find((rule) => rule.description === managedRuleDescription);
|
||||
if (current && comparableRule(current) === comparableRule(desired)) {
|
||||
return {changed: false, action: "unchanged", ruleId: current.id};
|
||||
}
|
||||
if (dryRun) {
|
||||
return {changed: true, action: current ? "update" : "create", desired};
|
||||
}
|
||||
if (!ruleset) {
|
||||
const created = await cloudflareRequest(`/zones/${zoneId}/rulesets`, {
|
||||
token,
|
||||
method: "POST",
|
||||
body: {
|
||||
name: "Xiteng label-managed cache rules",
|
||||
description: managedRuleDescription,
|
||||
kind: "zone",
|
||||
phase: "http_request_cache_settings",
|
||||
rules: [desired]
|
||||
}
|
||||
});
|
||||
return {changed: true, action: "create-ruleset", ruleId: created.rules?.[0]?.id};
|
||||
}
|
||||
if (!current) {
|
||||
const created = await cloudflareRequest(`/zones/${zoneId}/rulesets/${ruleset.id}/rules`, {
|
||||
token,
|
||||
method: "POST",
|
||||
body: desired
|
||||
});
|
||||
return {changed: true, action: "create", ruleId: created.id};
|
||||
}
|
||||
const updated = await cloudflareRequest(`/zones/${zoneId}/rulesets/${ruleset.id}/rules/${current.id}`, {
|
||||
token,
|
||||
method: "PATCH",
|
||||
body: desired
|
||||
});
|
||||
return {changed: true, action: "update", ruleId: updated.id};
|
||||
}
|
||||
|
||||
function writeIfChanged(path, value) {
|
||||
let current = "";
|
||||
try {
|
||||
current = readFileSync(path, "utf8");
|
||||
} catch {
|
||||
// The first discovery creates the file.
|
||||
}
|
||||
if (current === value) {
|
||||
return false;
|
||||
}
|
||||
const temporary = `${path}.tmp`;
|
||||
writeFileSync(temporary, value, {mode: 0o644});
|
||||
renameSync(temporary, path);
|
||||
return true;
|
||||
}
|
||||
|
||||
function dockerGet(socketPath, pathname) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = http.request({socketPath, path: pathname, method: "GET", headers: {Accept: "application/json"}}, (response) => {
|
||||
const chunks = [];
|
||||
response.on("data", (chunk) => chunks.push(chunk));
|
||||
response.on("end", () => {
|
||||
if ((response.statusCode || 500) >= 400) {
|
||||
reject(new Error(`Docker API ${response.statusCode}`));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
resolve(JSON.parse(Buffer.concat(chunks).toString("utf8")));
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
request.setTimeout(5000, () => request.destroy(new Error("Docker API timeout")));
|
||||
request.on("error", reject);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
function readToken(path) {
|
||||
try {
|
||||
return readFileSync(path, "utf8").trim();
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
export async function runController() {
|
||||
const port = boundedInteger(process.env.PORT, 8094, 1, 65535);
|
||||
const socketPath = process.env.DOCKER_SOCKET || "/var/run/docker.sock";
|
||||
const outputDirectory = process.env.OUTPUT_DIRECTORY || "/dynamic";
|
||||
const stateDirectory = process.env.STATE_DIRECTORY || "/state";
|
||||
const intervalMs = boundedInteger(process.env.DISCOVERY_INTERVAL_MS, 5000, 2000, 60000);
|
||||
const cloudflareIntervalMs = boundedInteger(process.env.CLOUDFLARE_SYNC_INTERVAL_MS, 60000, 10000, 3600000);
|
||||
const zoneName = process.env.CLOUDFLARE_ZONE_NAME || "xiteng.site";
|
||||
const zoneId = process.env.CLOUDFLARE_ZONE_ID || "";
|
||||
const tokenFile = process.env.CLOUDFLARE_API_TOKEN_FILE || "/run/secrets/cloudflare_cache_api_token";
|
||||
const traefikPath = `${outputDirectory}/xiteng-static-cache.yml`;
|
||||
const cloudflarePath = `${stateDirectory}/cloudflare-cache-rules.json`;
|
||||
const state = {
|
||||
ready: false,
|
||||
lastDiscoveryAt: null,
|
||||
policies: 0,
|
||||
routers: 0,
|
||||
errors: [],
|
||||
cloudflare: {status: "waiting-for-token", lastSyncAt: null, error: null}
|
||||
};
|
||||
let discoveryRunning = false;
|
||||
let lastCloudflareAttempt = 0;
|
||||
let lastCloudflareSpec = "";
|
||||
mkdirSync(outputDirectory, {recursive: true});
|
||||
mkdirSync(stateDirectory, {recursive: true});
|
||||
|
||||
async function discover() {
|
||||
if (discoveryRunning) {
|
||||
return;
|
||||
}
|
||||
discoveryRunning = true;
|
||||
try {
|
||||
const containers = await dockerGet(socketPath, "/containers/json?all=0");
|
||||
const built = buildConfiguration(containers, {zoneName});
|
||||
const traefikValue = `${JSON.stringify(built.traefik, null, 2)}\n`;
|
||||
const cloudflareValue = `${JSON.stringify(built.cloudflare, null, 2)}\n`;
|
||||
const changed = writeIfChanged(traefikPath, traefikValue);
|
||||
writeIfChanged(cloudflarePath, cloudflareValue);
|
||||
state.ready = true;
|
||||
state.lastDiscoveryAt = new Date().toISOString();
|
||||
state.policies = built.policyCount;
|
||||
state.routers = built.routerCount;
|
||||
state.errors = built.errors;
|
||||
if (changed || built.errors.length) {
|
||||
console.log(JSON.stringify({event: "cache-config", changed, policies: built.policyCount, routers: built.routerCount, errors: built.errors}));
|
||||
}
|
||||
|
||||
const token = readToken(tokenFile);
|
||||
const now = Date.now();
|
||||
if (!token) {
|
||||
state.cloudflare.status = "waiting-for-token";
|
||||
state.cloudflare.error = null;
|
||||
} else if (cloudflareValue !== lastCloudflareSpec || now - lastCloudflareAttempt >= cloudflareIntervalMs) {
|
||||
lastCloudflareAttempt = now;
|
||||
try {
|
||||
const result = await syncCloudflareRule(built.cloudflare, {zoneId, token});
|
||||
lastCloudflareSpec = cloudflareValue;
|
||||
state.cloudflare = {status: result.action, lastSyncAt: new Date().toISOString(), error: null};
|
||||
if (result.changed) {
|
||||
console.log(JSON.stringify({event: "cloudflare-cache-rule", action: result.action, policies: built.policyCount}));
|
||||
}
|
||||
} catch (error) {
|
||||
state.cloudflare = {status: "error", lastSyncAt: new Date().toISOString(), error: error.message};
|
||||
console.error(JSON.stringify({event: "cloudflare-cache-rule-error", error: error.message}));
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
state.errors = [error.message];
|
||||
console.error(JSON.stringify({event: "cache-controller-error", error: error.message}));
|
||||
} finally {
|
||||
discoveryRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
const server = http.createServer((request, response) => {
|
||||
if (request.url === "/healthz") {
|
||||
const healthy = state.ready && state.errors.length === 0 && state.cloudflare.status !== "error";
|
||||
response.writeHead(healthy ? 200 : 503, {"Content-Type": "application/json", "Cache-Control": "no-store"});
|
||||
response.end(JSON.stringify(state));
|
||||
return;
|
||||
}
|
||||
response.writeHead(404, {"Content-Type": "text/plain", "Cache-Control": "no-store"});
|
||||
response.end("Not found");
|
||||
});
|
||||
server.listen(port, "0.0.0.0");
|
||||
await discover();
|
||||
setInterval(discover, intervalMs).unref();
|
||||
}
|
||||
|
||||
if (process.argv[1] && new URL(import.meta.url).pathname === process.argv[1]) {
|
||||
runController().catch((error) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user