diff --git a/modules/network-checker/check/index.ts b/modules/network-checker/check/index.ts new file mode 100644 index 0000000..7c12a6a --- /dev/null +++ b/modules/network-checker/check/index.ts @@ -0,0 +1,84 @@ +// Dial everything the mesh claims is reachable, and say what was found (novox/hq ADR 0145). +// +// Runs on a cadence, from this machine, in this module's own container — the same position every other +// module on the machine calls from. That is the whole point: a check run by the host or by the control +// plane reaches these addresses by a path no ordinary caller uses, and would have passed throughout the +// outage that produced this module (novox/hq 04-ISSUES/145). +// +// It reports and does nothing else. A checker that repaired things would be a second control plane. + +import { readFileSync, writeFileSync, mkdirSync, renameSync } from "node:fs"; +import { dirname, join } from "node:path"; + +import { dial, tally, targetsFor, type Counts, type Result, type Roster } from "../reach.js"; + +/** Where the mesh renders this machine's view of the others, and where the counts are kept between runs. */ +const rosterFile = process.env.MESH_NETWORK_CHECKER_ROSTER ?? "/run/config/roster.json"; +const stateDir = process.env.MESH_NETWORK_CHECKER_STATE ?? "/run/state"; +const probePort = Number(process.env.MESH_NETWORK_CHECKER_PORT ?? "9876"); +const publicPort = process.env.MESH_NETWORK_CHECKER_PUBLIC_PORT + ? Number(process.env.MESH_NETWORK_CHECKER_PUBLIC_PORT) + : undefined; +const timeoutMs = Number(process.env.MESH_NETWORK_CHECKER_TIMEOUT_MS ?? "4000"); +const threshold = Number(process.env.MESH_NETWORK_CHECKER_THRESHOLD ?? "2"); + +/** read is a JSON file or a stated failure — never a silent default, which is how a checker comes to + * report that everything is fine because it read nothing. */ +function read(path: string, whenMissing: T | null): T { + try { + return JSON.parse(readFileSync(path, "utf8")) as T; + } catch (err) { + if (whenMissing !== null) return whenMissing; + console.error(`network-checker: cannot read ${path}: ${(err as Error).message}`); + process.exit(1); + } +} + +function writeAtomically(path: string, body: string): void { + mkdirSync(dirname(path), { recursive: true }); + const temp = `${path}.writing`; + writeFileSync(temp, body); + renameSync(temp, path); +} + +async function main(): Promise { + const roster = read(rosterFile, null); + if (!roster.machines?.length) { + console.error("network-checker: the roster names no machines; nothing to check"); + process.exit(1); + } + + const targets = targetsFor(roster, probePort, publicPort); + // In parallel, because a machine that is away should not delay the rest: a run that takes + // machines × timeout would outlast its own cadence on a mesh of any size. + const results: Result[] = await Promise.all(targets.map((t) => dial(t, timeoutMs))); + + const countsFile = join(stateDir, "consecutive.json"); + const { counts, broken } = tally(results, read(countsFile, {}), threshold); + writeAtomically(countsFile, JSON.stringify(counts, null, 1)); + + // Written whole, every run: a reader asking "what does this machine reach" gets an answer about now + // rather than the last time something changed. + writeAtomically(join(stateDir, "reach.json"), JSON.stringify({ + node: roster.node, + at: new Date().toISOString(), + checked: results.length, + broken: broken.length, + results, + }, null, 1)); + + for (const b of broken) { + console.error( + `network-checker: ${roster.node} cannot reach ${b.machine} (${b.claim}) at ${b.at}:${b.port} — ` + + `${b.failed} failed${b.detail ? `: ${b.detail}` : ""}, ${b.consecutive} run(s) running`); + } + if (broken.length === 0) { + console.log(`network-checker: ${roster.node} reaches all ${results.length} checked path(s)`); + } + + // A broken path is not this process failing. It did its job; exiting non-zero would make the mesh + // read the checker as the fault, and a scheduled step that fails is retried rather than believed. + process.exit(0); +} + +void main(); diff --git a/modules/network-checker/module.json b/modules/network-checker/module.json new file mode 100644 index 0000000..3cb623d --- /dev/null +++ b/modules/network-checker/module.json @@ -0,0 +1,61 @@ +{ + "module": "network-checker", + "version": "1", + "slug": "netcheck", + "listens": [ + { + "name": "probe", + "port": 9876, + "protocol": "tcp", + "from": "mesh", + "why": "what the other machines' checkers dial. Deliberately this module's own endpoint and nothing else's: it is admitted by exactly the rule that governs every internally-exposed service, so it fails when that rule is wrong. A probe on a port that is never closed — ssh, say — would have passed throughout the outage this module exists to catch (novox/hq ADR 0145)" + } + ], + "facts": { + "roster": { + "path": "/var/lib/network-checker/roster.json", + "template": "{\n \"generated\": \"by the mesh — do not edit; replaced whenever a machine joins or leaves\",\n \"node\": \"{{.Node}}\",\n \"machines\": [{{range $i, $m := .Machines}}{{if $i}},{{end}}\n { \"name\": \"{{$m.Name}}\", \"fqdn\": \"{{$m.FQDN}}\", \"address\": \"{{$m.Address}}\" }{{end}}\n ]\n}\n" + } + }, + "build": { + "artifacts": [ + { + "name": "code", + "kind": "bundle", + "language": "typescript", + "entrypoints": ["probe/index.js", "check/index.js"] + } + ] + }, + "resources": [ + { + "id": "state", + "type": "directory", + "path": "/var/lib/network-checker", + "mode": "0700" + }, + { + "id": "probe", + "type": "container", + "name": "mesh-network-checker-probe", + "args": ["run", "/app/modules/network-checker/dist/probe/index.js"], + "ports": ["9876"], + "state": "running", + "env": { "MESH_NETWORK_CHECKER_PORT": "9876" } + }, + { + "id": "check", + "type": "container", + "name": "mesh-network-checker-check", + "network": "host", + "schedule": "*/5 * * * *", + "args": ["run", "/app/modules/network-checker/dist/check/index.js"], + "volumes": ["/var/lib/network-checker:/run/state"], + "env": { + "MESH_NETWORK_CHECKER_ROSTER": "/run/state/roster.json", + "MESH_NETWORK_CHECKER_STATE": "/run/state", + "MESH_NETWORK_CHECKER_PORT": "${port:9876}" + } + } + ] +} diff --git a/modules/network-checker/package.json b/modules/network-checker/package.json new file mode 100644 index 0000000..8c85575 --- /dev/null +++ b/modules/network-checker/package.json @@ -0,0 +1,8 @@ +{ + "name": "@novox/module-network-checker", + "version": "0.1.0", + "description": "network-checker — dials what the mesh claims is reachable, from where the callers are, and says what it found.", + "type": "module", + "private": true, + "devDependencies": { "@types/node": "^22.0.0", "typescript": "^5.6.0" } +} diff --git a/modules/network-checker/probe/index.ts b/modules/network-checker/probe/index.ts new file mode 100644 index 0000000..cb95f28 --- /dev/null +++ b/modules/network-checker/probe/index.ts @@ -0,0 +1,31 @@ +// The endpoint the other machines' checkers dial (novox/hq ADR 0145). +// +// **This module's own endpoint is the instrument.** It is declared reachable over the private network +// like any other service, so it is admitted by exactly the rule that governs every internally-exposed +// service and it fails when that rule is wrong. A probe on a port that is never closed — ssh, say — +// would have passed throughout the outage this module exists to catch. +// +// It accepts a connection and closes it. Answering anything would make this a protocol, and then the +// question would be whether the protocol worked rather than whether the path did. + +import { createServer } from "node:net"; + +const port = Number(process.env.MESH_NETWORK_CHECKER_PORT ?? "9876"); + +const server = createServer((socket) => { + // Written before closing so a person dialling it by hand sees something, and so a half-open + // connection is not mistaken for a working path by a client that only checks the handshake. + socket.end("mesh network-checker\n"); +}); + +server.on("error", (err: Error) => { + // Said and fatal: a probe that cannot listen must not look like a probe that nothing dialled. + console.error(`network-checker: cannot serve the probe on ${port}: ${err.message}`); + process.exit(1); +}); + +server.listen(port, () => console.log(`network-checker: probe listening on ${port}`)); + +for (const signal of ["SIGTERM", "SIGINT"] as const) { + process.on(signal, () => server.close(() => process.exit(0))); +} diff --git a/modules/network-checker/reach.ts b/modules/network-checker/reach.ts new file mode 100644 index 0000000..4a8a16c --- /dev/null +++ b/modules/network-checker/reach.ts @@ -0,0 +1,155 @@ +// What the mesh claims is reachable, and how to find out (novox/hq ADR 0145). +// +// The mesh asserts three things are callable (ADR 0144): what runs on the same machine, another +// machine's service exposed to the private network, and another machine's service exposed publicly. +// This decides what to dial for each and reads the answers. It opens connections and nothing more — +// the module that owns a service is the one that knows whether it is working. +// +// **The target is this module's own endpoint, and that is deliberate.** The obvious thing to dial is a +// service every machine has, and the services every machine has are the ones never closed — ssh above +// all. Dialling one of those would have passed throughout the outage this exists to catch, because what +// broke was a service exposed to the private network and ssh is admitted unconditionally. A probe on a +// port that cannot fail measures nothing. + +import { connect } from "node:net"; +import { lookup } from "node:dns"; + +/** One machine as the mesh's roster describes it. */ +export interface Machine { + name: string; + fqdn: string; + address: string; + /** The name this machine is reached by from outside, where it has one. Absent for most machines, and + * a machine with no public face has no public claim to check. */ + public?: string; +} + +/** The roster the mesh renders for this module: who this machine is, and who the others are. */ +export interface Roster { + node: string; + machines: Machine[]; +} + +/** Which of the mesh's three claims a check is about, so a failure says which one broke. */ +export type Claim = "this machine" | "the private network" | "the public network"; + +/** One thing to dial. */ +export interface Target { + claim: Claim; + machine: string; + /** What to dial — a name where the point is that names resolve, an address where it is not. */ + at: string; + port: number; + /** Whether `at` is a name that must resolve first, so a resolution failure is reported as one. */ + byName: boolean; +} + +/** What one dial found. */ +export interface Result extends Target { + ok: boolean; + /** Which step failed, so a reader is sent to the right place: the resolver, or the filter. */ + failed?: "resolution" | "connection"; + detail?: string; + ms: number; +} + +/** + * targetsFor is everything this machine should be able to reach, from the roster it was given. + * + * Its own machine first, because that is the case that distinguishes a caller on the machine from a + * caller in one of its containers — the one that broke. Then every other machine over the private + * network. The public claim is only checked where a public address is known for a machine, because a + * machine with no public face has nothing to fail. + */ +export function targetsFor(roster: Roster, probePort: number, publicPort?: number): Target[] { + const out: Target[] = []; + for (const m of roster.machines) { + const own = m.name === roster.node; + out.push({ + claim: own ? "this machine" : "the private network", + machine: m.name, + at: m.address, + port: probePort, + byName: false, + }); + // And by name, because a name that does not resolve and a port that does not answer are different + // faults with different owners. + out.push({ + claim: own ? "this machine" : "the private network", + machine: m.name, + at: m.fqdn, + port: probePort, + byName: true, + }); + } + if (publicPort !== undefined) { + for (const m of roster.machines) { + if (!m.public) continue; + out.push({ + claim: "the public network", + machine: m.name, + at: m.public, + port: publicPort, + byName: true, + }); + } + } + return out; +} + +/** dial opens a connection and closes it. Whether the port accepts is the whole of what is asked. */ +export function dial(target: Target, timeoutMs: number): Promise { + const began = Date.now(); + const done = (ok: boolean, failed?: Result["failed"], detail?: string): Result => ({ + ...target, ok, failed, detail, ms: Date.now() - began, + }); + + return new Promise((resolve) => { + const open = () => { + const socket = connect({ host: target.at, port: target.port }); + const finish = (r: Result) => { socket.destroy(); resolve(r); }; + socket.setTimeout(timeoutMs); + socket.once("connect", () => finish(done(true))); + socket.once("timeout", () => finish(done(false, "connection", "timed out"))); + socket.once("error", (err: Error) => finish(done(false, "connection", err.message))); + }; + + if (!target.byName) { open(); return; } + // Resolved first and reported separately: a checker that says "unreachable" for a name the + // resolver never answered sends a reader to the filter, which is not where the fault is. + lookup(target.at, (err) => { + if (err) { resolve(done(false, "resolution", err.message)); return; } + open(); + }); + }); +} + +/** A path's running count of consecutive failures, keyed so it survives between runs. */ +export type Counts = Record; + +/** keyOf names one path, stably, so a count follows it across runs. */ +export function keyOf(t: Target): string { + return `${t.claim}|${t.machine}|${t.at}|${t.port}`; +} + +/** + * tally folds this run's results into the counts carried from the last one. + * + * **One failure is not a fault.** A machine rebooting is ordinary, and a checker that cries at the + * first missed dial trains a reader to ignore it — which is worse than not checking (ADR 0145). A path + * is broken once it has failed on consecutive runs, and the count travels with the result so a reader + * can tell "briefly away" from "never worked". + */ +export function tally(results: Result[], before: Counts, threshold: number): { + counts: Counts; broken: Array; +} { + const counts: Counts = {}; + const broken: Array = []; + for (const r of results) { + const key = keyOf(r); + const n = r.ok ? 0 : (before[key] ?? 0) + 1; + if (n > 0) counts[key] = n; + if (n >= threshold) broken.push({ ...r, consecutive: n }); + } + return { counts, broken }; +} diff --git a/modules/network-checker/test/reach.test.ts b/modules/network-checker/test/reach.test.ts new file mode 100644 index 0000000..875e8e0 --- /dev/null +++ b/modules/network-checker/test/reach.test.ts @@ -0,0 +1,72 @@ +import { strict as assert } from "node:assert"; +import test from "node:test"; + +import { keyOf, tally, targetsFor, type Result, type Roster } from "../reach.js"; + +const roster: Roster = { + node: "here", + machines: [ + { name: "here", fqdn: "here.internal", address: "10.0.0.1" }, + { name: "there", fqdn: "there.internal", address: "10.0.0.2", public: "there.example.test" }, + ], +}; + +test("its own machine is checked, which is the case that distinguishes a caller on it from one in a container", () => { + const own = targetsFor(roster, 9876).filter((t) => t.claim === "this machine"); + assert.equal(own.length, 2, "its own machine by address and by name"); + assert.ok(own.some((t) => t.at === "10.0.0.1" && !t.byName)); + assert.ok(own.some((t) => t.at === "here.internal" && t.byName)); +}); + +test("every other machine is checked over the private network", () => { + const other = targetsFor(roster, 9876).filter((t) => t.claim === "the private network"); + assert.deepEqual(other.map((t) => t.machine), ["there", "there"]); +}); + +test("the public claim is only checked where a machine has a public name", () => { + const pub = targetsFor(roster, 9876, 443).filter((t) => t.claim === "the public network"); + assert.equal(pub.length, 1, "only the machine with a public name"); + assert.equal(pub[0]!.at, "there.example.test"); + assert.equal(pub[0]!.port, 443); +}); + +test("no public claim is made when no public port was given", () => { + assert.equal(targetsFor(roster, 9876).filter((t) => t.claim === "the public network").length, 0); +}); + +const failed = (at: string): Result => ({ + claim: "this machine", machine: "here", at, port: 9876, byName: false, + ok: false, failed: "connection", ms: 1, +}); +const passed = (at: string): Result => ({ + claim: "this machine", machine: "here", at, port: 9876, byName: false, ok: true, ms: 1, +}); + +test("one failure is not a fault — a machine rebooting is ordinary", () => { + const { counts, broken } = tally([failed("10.0.0.1")], {}, 2); + assert.equal(broken.length, 0, "one missed dial says nothing"); + assert.equal(counts[keyOf(failed("10.0.0.1"))], 1, "and is remembered"); +}); + +test("a path that keeps failing is broken, and the count travels with it", () => { + const first = tally([failed("10.0.0.1")], {}, 2); + const second = tally([failed("10.0.0.1")], first.counts, 2); + assert.equal(second.broken.length, 1); + assert.equal(second.broken[0]!.consecutive, 2, "so a reader can tell briefly away from never worked"); +}); + +test("a path that recovers stops being counted", () => { + const first = tally([failed("10.0.0.1")], {}, 2); + const second = tally([passed("10.0.0.1")], first.counts, 2); + assert.equal(second.broken.length, 0); + assert.deepEqual(second.counts, {}, "nothing carried forward for a path that works"); +}); + +test("a count follows one path and not another", () => { + const a = failed("10.0.0.1"); + const b = failed("10.0.0.2"); + const first = tally([a, b], {}, 2); + const second = tally([a], first.counts, 2); + assert.equal(second.broken.length, 1, "only the path dialled this run is judged"); + assert.equal(second.broken[0]!.at, "10.0.0.1"); +}); diff --git a/modules/network-checker/tsconfig.json b/modules/network-checker/tsconfig.json new file mode 100644 index 0000000..a6d0744 --- /dev/null +++ b/modules/network-checker/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["reach.ts", "probe/index.ts", "check/index.ts", "test/*.ts"] +}