diff --git a/modules/network-checker/check/index.ts b/modules/network-checker/check/index.ts deleted file mode 100644 index 7c12a6a..0000000 --- a/modules/network-checker/check/index.ts +++ /dev/null @@ -1,84 +0,0 @@ -// Dial everything the mesh claims is reachable, and say what was found (novox/hq ADR 0145). -// -// Runs on a cadence, from this machine, in this module's own container — the same position every other -// module on the machine calls from. That is the whole point: a check run by the host or by the control -// plane reaches these addresses by a path no ordinary caller uses, and would have passed throughout the -// outage that produced this module (novox/hq 04-ISSUES/145). -// -// It reports and does nothing else. A checker that repaired things would be a second control plane. - -import { readFileSync, writeFileSync, mkdirSync, renameSync } from "node:fs"; -import { dirname, join } from "node:path"; - -import { dial, tally, targetsFor, type Counts, type Result, type Roster } from "../reach.js"; - -/** Where the mesh renders this machine's view of the others, and where the counts are kept between runs. */ -const rosterFile = process.env.MESH_NETWORK_CHECKER_ROSTER ?? "/run/config/roster.json"; -const stateDir = process.env.MESH_NETWORK_CHECKER_STATE ?? "/run/state"; -const probePort = Number(process.env.MESH_NETWORK_CHECKER_PORT ?? "9876"); -const publicPort = process.env.MESH_NETWORK_CHECKER_PUBLIC_PORT - ? Number(process.env.MESH_NETWORK_CHECKER_PUBLIC_PORT) - : undefined; -const timeoutMs = Number(process.env.MESH_NETWORK_CHECKER_TIMEOUT_MS ?? "4000"); -const threshold = Number(process.env.MESH_NETWORK_CHECKER_THRESHOLD ?? "2"); - -/** read is a JSON file or a stated failure — never a silent default, which is how a checker comes to - * report that everything is fine because it read nothing. */ -function read(path: string, whenMissing: T | null): T { - try { - return JSON.parse(readFileSync(path, "utf8")) as T; - } catch (err) { - if (whenMissing !== null) return whenMissing; - console.error(`network-checker: cannot read ${path}: ${(err as Error).message}`); - process.exit(1); - } -} - -function writeAtomically(path: string, body: string): void { - mkdirSync(dirname(path), { recursive: true }); - const temp = `${path}.writing`; - writeFileSync(temp, body); - renameSync(temp, path); -} - -async function main(): Promise { - const roster = read(rosterFile, null); - if (!roster.machines?.length) { - console.error("network-checker: the roster names no machines; nothing to check"); - process.exit(1); - } - - const targets = targetsFor(roster, probePort, publicPort); - // In parallel, because a machine that is away should not delay the rest: a run that takes - // machines × timeout would outlast its own cadence on a mesh of any size. - const results: Result[] = await Promise.all(targets.map((t) => dial(t, timeoutMs))); - - const countsFile = join(stateDir, "consecutive.json"); - const { counts, broken } = tally(results, read(countsFile, {}), threshold); - writeAtomically(countsFile, JSON.stringify(counts, null, 1)); - - // Written whole, every run: a reader asking "what does this machine reach" gets an answer about now - // rather than the last time something changed. - writeAtomically(join(stateDir, "reach.json"), JSON.stringify({ - node: roster.node, - at: new Date().toISOString(), - checked: results.length, - broken: broken.length, - results, - }, null, 1)); - - for (const b of broken) { - console.error( - `network-checker: ${roster.node} cannot reach ${b.machine} (${b.claim}) at ${b.at}:${b.port} — ` + - `${b.failed} failed${b.detail ? `: ${b.detail}` : ""}, ${b.consecutive} run(s) running`); - } - if (broken.length === 0) { - console.log(`network-checker: ${roster.node} reaches all ${results.length} checked path(s)`); - } - - // A broken path is not this process failing. It did its job; exiting non-zero would make the mesh - // read the checker as the fault, and a scheduled step that fails is retried rather than believed. - process.exit(0); -} - -void main(); diff --git a/modules/network-checker/module.json b/modules/network-checker/module.json deleted file mode 100644 index 3cb623d..0000000 --- a/modules/network-checker/module.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "module": "network-checker", - "version": "1", - "slug": "netcheck", - "listens": [ - { - "name": "probe", - "port": 9876, - "protocol": "tcp", - "from": "mesh", - "why": "what the other machines' checkers dial. Deliberately this module's own endpoint and nothing else's: it is admitted by exactly the rule that governs every internally-exposed service, so it fails when that rule is wrong. A probe on a port that is never closed — ssh, say — would have passed throughout the outage this module exists to catch (novox/hq ADR 0145)" - } - ], - "facts": { - "roster": { - "path": "/var/lib/network-checker/roster.json", - "template": "{\n \"generated\": \"by the mesh — do not edit; replaced whenever a machine joins or leaves\",\n \"node\": \"{{.Node}}\",\n \"machines\": [{{range $i, $m := .Machines}}{{if $i}},{{end}}\n { \"name\": \"{{$m.Name}}\", \"fqdn\": \"{{$m.FQDN}}\", \"address\": \"{{$m.Address}}\" }{{end}}\n ]\n}\n" - } - }, - "build": { - "artifacts": [ - { - "name": "code", - "kind": "bundle", - "language": "typescript", - "entrypoints": ["probe/index.js", "check/index.js"] - } - ] - }, - "resources": [ - { - "id": "state", - "type": "directory", - "path": "/var/lib/network-checker", - "mode": "0700" - }, - { - "id": "probe", - "type": "container", - "name": "mesh-network-checker-probe", - "args": ["run", "/app/modules/network-checker/dist/probe/index.js"], - "ports": ["9876"], - "state": "running", - "env": { "MESH_NETWORK_CHECKER_PORT": "9876" } - }, - { - "id": "check", - "type": "container", - "name": "mesh-network-checker-check", - "network": "host", - "schedule": "*/5 * * * *", - "args": ["run", "/app/modules/network-checker/dist/check/index.js"], - "volumes": ["/var/lib/network-checker:/run/state"], - "env": { - "MESH_NETWORK_CHECKER_ROSTER": "/run/state/roster.json", - "MESH_NETWORK_CHECKER_STATE": "/run/state", - "MESH_NETWORK_CHECKER_PORT": "${port:9876}" - } - } - ] -} diff --git a/modules/network-checker/package.json b/modules/network-checker/package.json deleted file mode 100644 index 8c85575..0000000 --- a/modules/network-checker/package.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "@novox/module-network-checker", - "version": "0.1.0", - "description": "network-checker — dials what the mesh claims is reachable, from where the callers are, and says what it found.", - "type": "module", - "private": true, - "devDependencies": { "@types/node": "^22.0.0", "typescript": "^5.6.0" } -} diff --git a/modules/network-checker/probe/index.ts b/modules/network-checker/probe/index.ts deleted file mode 100644 index cb95f28..0000000 --- a/modules/network-checker/probe/index.ts +++ /dev/null @@ -1,31 +0,0 @@ -// The endpoint the other machines' checkers dial (novox/hq ADR 0145). -// -// **This module's own endpoint is the instrument.** It is declared reachable over the private network -// like any other service, so it is admitted by exactly the rule that governs every internally-exposed -// service and it fails when that rule is wrong. A probe on a port that is never closed — ssh, say — -// would have passed throughout the outage this module exists to catch. -// -// It accepts a connection and closes it. Answering anything would make this a protocol, and then the -// question would be whether the protocol worked rather than whether the path did. - -import { createServer } from "node:net"; - -const port = Number(process.env.MESH_NETWORK_CHECKER_PORT ?? "9876"); - -const server = createServer((socket) => { - // Written before closing so a person dialling it by hand sees something, and so a half-open - // connection is not mistaken for a working path by a client that only checks the handshake. - socket.end("mesh network-checker\n"); -}); - -server.on("error", (err: Error) => { - // Said and fatal: a probe that cannot listen must not look like a probe that nothing dialled. - console.error(`network-checker: cannot serve the probe on ${port}: ${err.message}`); - process.exit(1); -}); - -server.listen(port, () => console.log(`network-checker: probe listening on ${port}`)); - -for (const signal of ["SIGTERM", "SIGINT"] as const) { - process.on(signal, () => server.close(() => process.exit(0))); -} diff --git a/modules/network-checker/reach.ts b/modules/network-checker/reach.ts deleted file mode 100644 index 4a8a16c..0000000 --- a/modules/network-checker/reach.ts +++ /dev/null @@ -1,155 +0,0 @@ -// What the mesh claims is reachable, and how to find out (novox/hq ADR 0145). -// -// The mesh asserts three things are callable (ADR 0144): what runs on the same machine, another -// machine's service exposed to the private network, and another machine's service exposed publicly. -// This decides what to dial for each and reads the answers. It opens connections and nothing more — -// the module that owns a service is the one that knows whether it is working. -// -// **The target is this module's own endpoint, and that is deliberate.** The obvious thing to dial is a -// service every machine has, and the services every machine has are the ones never closed — ssh above -// all. Dialling one of those would have passed throughout the outage this exists to catch, because what -// broke was a service exposed to the private network and ssh is admitted unconditionally. A probe on a -// port that cannot fail measures nothing. - -import { connect } from "node:net"; -import { lookup } from "node:dns"; - -/** One machine as the mesh's roster describes it. */ -export interface Machine { - name: string; - fqdn: string; - address: string; - /** The name this machine is reached by from outside, where it has one. Absent for most machines, and - * a machine with no public face has no public claim to check. */ - public?: string; -} - -/** The roster the mesh renders for this module: who this machine is, and who the others are. */ -export interface Roster { - node: string; - machines: Machine[]; -} - -/** Which of the mesh's three claims a check is about, so a failure says which one broke. */ -export type Claim = "this machine" | "the private network" | "the public network"; - -/** One thing to dial. */ -export interface Target { - claim: Claim; - machine: string; - /** What to dial — a name where the point is that names resolve, an address where it is not. */ - at: string; - port: number; - /** Whether `at` is a name that must resolve first, so a resolution failure is reported as one. */ - byName: boolean; -} - -/** What one dial found. */ -export interface Result extends Target { - ok: boolean; - /** Which step failed, so a reader is sent to the right place: the resolver, or the filter. */ - failed?: "resolution" | "connection"; - detail?: string; - ms: number; -} - -/** - * targetsFor is everything this machine should be able to reach, from the roster it was given. - * - * Its own machine first, because that is the case that distinguishes a caller on the machine from a - * caller in one of its containers — the one that broke. Then every other machine over the private - * network. The public claim is only checked where a public address is known for a machine, because a - * machine with no public face has nothing to fail. - */ -export function targetsFor(roster: Roster, probePort: number, publicPort?: number): Target[] { - const out: Target[] = []; - for (const m of roster.machines) { - const own = m.name === roster.node; - out.push({ - claim: own ? "this machine" : "the private network", - machine: m.name, - at: m.address, - port: probePort, - byName: false, - }); - // And by name, because a name that does not resolve and a port that does not answer are different - // faults with different owners. - out.push({ - claim: own ? "this machine" : "the private network", - machine: m.name, - at: m.fqdn, - port: probePort, - byName: true, - }); - } - if (publicPort !== undefined) { - for (const m of roster.machines) { - if (!m.public) continue; - out.push({ - claim: "the public network", - machine: m.name, - at: m.public, - port: publicPort, - byName: true, - }); - } - } - return out; -} - -/** dial opens a connection and closes it. Whether the port accepts is the whole of what is asked. */ -export function dial(target: Target, timeoutMs: number): Promise { - const began = Date.now(); - const done = (ok: boolean, failed?: Result["failed"], detail?: string): Result => ({ - ...target, ok, failed, detail, ms: Date.now() - began, - }); - - return new Promise((resolve) => { - const open = () => { - const socket = connect({ host: target.at, port: target.port }); - const finish = (r: Result) => { socket.destroy(); resolve(r); }; - socket.setTimeout(timeoutMs); - socket.once("connect", () => finish(done(true))); - socket.once("timeout", () => finish(done(false, "connection", "timed out"))); - socket.once("error", (err: Error) => finish(done(false, "connection", err.message))); - }; - - if (!target.byName) { open(); return; } - // Resolved first and reported separately: a checker that says "unreachable" for a name the - // resolver never answered sends a reader to the filter, which is not where the fault is. - lookup(target.at, (err) => { - if (err) { resolve(done(false, "resolution", err.message)); return; } - open(); - }); - }); -} - -/** A path's running count of consecutive failures, keyed so it survives between runs. */ -export type Counts = Record; - -/** keyOf names one path, stably, so a count follows it across runs. */ -export function keyOf(t: Target): string { - return `${t.claim}|${t.machine}|${t.at}|${t.port}`; -} - -/** - * tally folds this run's results into the counts carried from the last one. - * - * **One failure is not a fault.** A machine rebooting is ordinary, and a checker that cries at the - * first missed dial trains a reader to ignore it — which is worse than not checking (ADR 0145). A path - * is broken once it has failed on consecutive runs, and the count travels with the result so a reader - * can tell "briefly away" from "never worked". - */ -export function tally(results: Result[], before: Counts, threshold: number): { - counts: Counts; broken: Array; -} { - const counts: Counts = {}; - const broken: Array = []; - for (const r of results) { - const key = keyOf(r); - const n = r.ok ? 0 : (before[key] ?? 0) + 1; - if (n > 0) counts[key] = n; - if (n >= threshold) broken.push({ ...r, consecutive: n }); - } - return { counts, broken }; -} diff --git a/modules/network-checker/test/reach.test.ts b/modules/network-checker/test/reach.test.ts deleted file mode 100644 index 875e8e0..0000000 --- a/modules/network-checker/test/reach.test.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { strict as assert } from "node:assert"; -import test from "node:test"; - -import { keyOf, tally, targetsFor, type Result, type Roster } from "../reach.js"; - -const roster: Roster = { - node: "here", - machines: [ - { name: "here", fqdn: "here.internal", address: "10.0.0.1" }, - { name: "there", fqdn: "there.internal", address: "10.0.0.2", public: "there.example.test" }, - ], -}; - -test("its own machine is checked, which is the case that distinguishes a caller on it from one in a container", () => { - const own = targetsFor(roster, 9876).filter((t) => t.claim === "this machine"); - assert.equal(own.length, 2, "its own machine by address and by name"); - assert.ok(own.some((t) => t.at === "10.0.0.1" && !t.byName)); - assert.ok(own.some((t) => t.at === "here.internal" && t.byName)); -}); - -test("every other machine is checked over the private network", () => { - const other = targetsFor(roster, 9876).filter((t) => t.claim === "the private network"); - assert.deepEqual(other.map((t) => t.machine), ["there", "there"]); -}); - -test("the public claim is only checked where a machine has a public name", () => { - const pub = targetsFor(roster, 9876, 443).filter((t) => t.claim === "the public network"); - assert.equal(pub.length, 1, "only the machine with a public name"); - assert.equal(pub[0]!.at, "there.example.test"); - assert.equal(pub[0]!.port, 443); -}); - -test("no public claim is made when no public port was given", () => { - assert.equal(targetsFor(roster, 9876).filter((t) => t.claim === "the public network").length, 0); -}); - -const failed = (at: string): Result => ({ - claim: "this machine", machine: "here", at, port: 9876, byName: false, - ok: false, failed: "connection", ms: 1, -}); -const passed = (at: string): Result => ({ - claim: "this machine", machine: "here", at, port: 9876, byName: false, ok: true, ms: 1, -}); - -test("one failure is not a fault — a machine rebooting is ordinary", () => { - const { counts, broken } = tally([failed("10.0.0.1")], {}, 2); - assert.equal(broken.length, 0, "one missed dial says nothing"); - assert.equal(counts[keyOf(failed("10.0.0.1"))], 1, "and is remembered"); -}); - -test("a path that keeps failing is broken, and the count travels with it", () => { - const first = tally([failed("10.0.0.1")], {}, 2); - const second = tally([failed("10.0.0.1")], first.counts, 2); - assert.equal(second.broken.length, 1); - assert.equal(second.broken[0]!.consecutive, 2, "so a reader can tell briefly away from never worked"); -}); - -test("a path that recovers stops being counted", () => { - const first = tally([failed("10.0.0.1")], {}, 2); - const second = tally([passed("10.0.0.1")], first.counts, 2); - assert.equal(second.broken.length, 0); - assert.deepEqual(second.counts, {}, "nothing carried forward for a path that works"); -}); - -test("a count follows one path and not another", () => { - const a = failed("10.0.0.1"); - const b = failed("10.0.0.2"); - const first = tally([a, b], {}, 2); - const second = tally([a], first.counts, 2); - assert.equal(second.broken.length, 1, "only the path dialled this run is judged"); - assert.equal(second.broken[0]!.at, "10.0.0.1"); -}); diff --git a/modules/network-checker/tsconfig.json b/modules/network-checker/tsconfig.json deleted file mode 100644 index a6d0744..0000000 --- a/modules/network-checker/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "NodeNext", - "moduleResolution": "NodeNext", - "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "noEmit": true - }, - "include": ["reach.ts", "probe/index.ts", "check/index.ts", "test/*.ts"] -}