// What the mesh claims is reachable, and how to find out (novox/hq ADR 0145). // // The mesh asserts three things are callable (ADR 0144): what runs on the same machine, another // machine's service exposed to the private network, and another machine's service exposed publicly. // This decides what to dial for each and reads the answers. It opens connections and nothing more — // the module that owns a service is the one that knows whether it is working. // // **The target is this module's own endpoint, and that is deliberate.** The obvious thing to dial is a // service every machine has, and the services every machine has are the ones never closed — ssh above // all. Dialling one of those would have passed throughout the outage this exists to catch, because what // broke was a service exposed to the private network and ssh is admitted unconditionally. A probe on a // port that cannot fail measures nothing. import { connect } from "node:net"; import { lookup } from "node:dns"; /** One machine as the mesh's roster describes it. */ export interface Machine { name: string; fqdn: string; address: string; /** The name this machine is reached by from outside, where it has one. Absent for most machines, and * a machine with no public face has no public claim to check. */ public?: string; } /** The roster the mesh renders for this module: who this machine is, and who the others are. */ export interface Roster { node: string; machines: Machine[]; } /** Which of the mesh's three claims a check is about, so a failure says which one broke. */ export type Claim = "this machine" | "the private network" | "the public network"; /** One thing to dial. */ export interface Target { claim: Claim; machine: string; /** What to dial — a name where the point is that names resolve, an address where it is not. */ at: string; port: number; /** Whether `at` is a name that must resolve first, so a resolution failure is reported as one. */ byName: boolean; } /** What one dial found. */ export interface Result extends Target { ok: boolean; /** Which step failed, so a reader is sent to the right place: the resolver, or the filter. */ failed?: "resolution" | "connection"; detail?: string; ms: number; } /** * targetsFor is everything this machine should be able to reach, from the roster it was given. * * Its own machine first, because that is the case that distinguishes a caller on the machine from a * caller in one of its containers — the one that broke. Then every other machine over the private * network. The public claim is only checked where a public address is known for a machine, because a * machine with no public face has nothing to fail. */ export function targetsFor(roster: Roster, probePort: number, publicPort?: number): Target[] { const out: Target[] = []; for (const m of roster.machines) { const own = m.name === roster.node; out.push({ claim: own ? "this machine" : "the private network", machine: m.name, at: m.address, port: probePort, byName: false, }); // And by name, because a name that does not resolve and a port that does not answer are different // faults with different owners. out.push({ claim: own ? "this machine" : "the private network", machine: m.name, at: m.fqdn, port: probePort, byName: true, }); } if (publicPort !== undefined) { for (const m of roster.machines) { if (!m.public) continue; out.push({ claim: "the public network", machine: m.name, at: m.public, port: publicPort, byName: true, }); } } return out; } /** dial opens a connection and closes it. Whether the port accepts is the whole of what is asked. */ export function dial(target: Target, timeoutMs: number): Promise { const began = Date.now(); const done = (ok: boolean, failed?: Result["failed"], detail?: string): Result => ({ ...target, ok, failed, detail, ms: Date.now() - began, }); return new Promise((resolve) => { const open = () => { const socket = connect({ host: target.at, port: target.port }); const finish = (r: Result) => { socket.destroy(); resolve(r); }; socket.setTimeout(timeoutMs); socket.once("connect", () => finish(done(true))); socket.once("timeout", () => finish(done(false, "connection", "timed out"))); socket.once("error", (err: Error) => finish(done(false, "connection", err.message))); }; if (!target.byName) { open(); return; } // Resolved first and reported separately: a checker that says "unreachable" for a name the // resolver never answered sends a reader to the filter, which is not where the fault is. lookup(target.at, (err) => { if (err) { resolve(done(false, "resolution", err.message)); return; } open(); }); }); } /** A path's running count of consecutive failures, keyed so it survives between runs. */ export type Counts = Record; /** keyOf names one path, stably, so a count follows it across runs. */ export function keyOf(t: Target): string { return `${t.claim}|${t.machine}|${t.at}|${t.port}`; } /** * tally folds this run's results into the counts carried from the last one. * * **One failure is not a fault.** A machine rebooting is ordinary, and a checker that cries at the * first missed dial trains a reader to ignore it — which is worse than not checking (ADR 0145). A path * is broken once it has failed on consecutive runs, and the count travels with the result so a reader * can tell "briefly away" from "never worked". */ export function tally(results: Result[], before: Counts, threshold: number): { counts: Counts; broken: Array; } { const counts: Counts = {}; const broken: Array = []; for (const r of results) { const key = keyOf(r); const n = r.ok ? 0 : (before[key] ?? 0) + 1; if (n > 0) counts[key] = n; if (n >= threshold) broken.push({ ...r, consecutive: n }); } return { counts, broken }; }