Remove the network-checker module: it does not do what was decided #141
@@ -1,84 +0,0 @@
|
||||
// Dial everything the mesh claims is reachable, and say what was found (novox/hq ADR 0145).
|
||||
//
|
||||
// Runs on a cadence, from this machine, in this module's own container — the same position every other
|
||||
// module on the machine calls from. That is the whole point: a check run by the host or by the control
|
||||
// plane reaches these addresses by a path no ordinary caller uses, and would have passed throughout the
|
||||
// outage that produced this module (novox/hq 04-ISSUES/145).
|
||||
//
|
||||
// It reports and does nothing else. A checker that repaired things would be a second control plane.
|
||||
|
||||
import { readFileSync, writeFileSync, mkdirSync, renameSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
|
||||
import { dial, tally, targetsFor, type Counts, type Result, type Roster } from "../reach.js";
|
||||
|
||||
/** Where the mesh renders this machine's view of the others, and where the counts are kept between runs. */
|
||||
const rosterFile = process.env.MESH_NETWORK_CHECKER_ROSTER ?? "/run/config/roster.json";
|
||||
const stateDir = process.env.MESH_NETWORK_CHECKER_STATE ?? "/run/state";
|
||||
const probePort = Number(process.env.MESH_NETWORK_CHECKER_PORT ?? "9876");
|
||||
const publicPort = process.env.MESH_NETWORK_CHECKER_PUBLIC_PORT
|
||||
? Number(process.env.MESH_NETWORK_CHECKER_PUBLIC_PORT)
|
||||
: undefined;
|
||||
const timeoutMs = Number(process.env.MESH_NETWORK_CHECKER_TIMEOUT_MS ?? "4000");
|
||||
const threshold = Number(process.env.MESH_NETWORK_CHECKER_THRESHOLD ?? "2");
|
||||
|
||||
/** read is a JSON file or a stated failure — never a silent default, which is how a checker comes to
|
||||
* report that everything is fine because it read nothing. */
|
||||
function read<T>(path: string, whenMissing: T | null): T {
|
||||
try {
|
||||
return JSON.parse(readFileSync(path, "utf8")) as T;
|
||||
} catch (err) {
|
||||
if (whenMissing !== null) return whenMissing;
|
||||
console.error(`network-checker: cannot read ${path}: ${(err as Error).message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
function writeAtomically(path: string, body: string): void {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
const temp = `${path}.writing`;
|
||||
writeFileSync(temp, body);
|
||||
renameSync(temp, path);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const roster = read<Roster>(rosterFile, null);
|
||||
if (!roster.machines?.length) {
|
||||
console.error("network-checker: the roster names no machines; nothing to check");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const targets = targetsFor(roster, probePort, publicPort);
|
||||
// In parallel, because a machine that is away should not delay the rest: a run that takes
|
||||
// machines × timeout would outlast its own cadence on a mesh of any size.
|
||||
const results: Result[] = await Promise.all(targets.map((t) => dial(t, timeoutMs)));
|
||||
|
||||
const countsFile = join(stateDir, "consecutive.json");
|
||||
const { counts, broken } = tally(results, read<Counts>(countsFile, {}), threshold);
|
||||
writeAtomically(countsFile, JSON.stringify(counts, null, 1));
|
||||
|
||||
// Written whole, every run: a reader asking "what does this machine reach" gets an answer about now
|
||||
// rather than the last time something changed.
|
||||
writeAtomically(join(stateDir, "reach.json"), JSON.stringify({
|
||||
node: roster.node,
|
||||
at: new Date().toISOString(),
|
||||
checked: results.length,
|
||||
broken: broken.length,
|
||||
results,
|
||||
}, null, 1));
|
||||
|
||||
for (const b of broken) {
|
||||
console.error(
|
||||
`network-checker: ${roster.node} cannot reach ${b.machine} (${b.claim}) at ${b.at}:${b.port} — ` +
|
||||
`${b.failed} failed${b.detail ? `: ${b.detail}` : ""}, ${b.consecutive} run(s) running`);
|
||||
}
|
||||
if (broken.length === 0) {
|
||||
console.log(`network-checker: ${roster.node} reaches all ${results.length} checked path(s)`);
|
||||
}
|
||||
|
||||
// A broken path is not this process failing. It did its job; exiting non-zero would make the mesh
|
||||
// read the checker as the fault, and a scheduled step that fails is retried rather than believed.
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
void main();
|
||||
@@ -1,61 +0,0 @@
|
||||
{
|
||||
"module": "network-checker",
|
||||
"version": "1",
|
||||
"slug": "netcheck",
|
||||
"listens": [
|
||||
{
|
||||
"name": "probe",
|
||||
"port": 9876,
|
||||
"protocol": "tcp",
|
||||
"from": "mesh",
|
||||
"why": "what the other machines' checkers dial. Deliberately this module's own endpoint and nothing else's: it is admitted by exactly the rule that governs every internally-exposed service, so it fails when that rule is wrong. A probe on a port that is never closed — ssh, say — would have passed throughout the outage this module exists to catch (novox/hq ADR 0145)"
|
||||
}
|
||||
],
|
||||
"facts": {
|
||||
"roster": {
|
||||
"path": "/var/lib/network-checker/roster.json",
|
||||
"template": "{\n \"generated\": \"by the mesh — do not edit; replaced whenever a machine joins or leaves\",\n \"node\": \"{{.Node}}\",\n \"machines\": [{{range $i, $m := .Machines}}{{if $i}},{{end}}\n { \"name\": \"{{$m.Name}}\", \"fqdn\": \"{{$m.FQDN}}\", \"address\": \"{{$m.Address}}\" }{{end}}\n ]\n}\n"
|
||||
}
|
||||
},
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "code",
|
||||
"kind": "bundle",
|
||||
"language": "typescript",
|
||||
"entrypoints": ["probe/index.js", "check/index.js"]
|
||||
}
|
||||
]
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/network-checker",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "probe",
|
||||
"type": "container",
|
||||
"name": "mesh-network-checker-probe",
|
||||
"args": ["run", "/app/modules/network-checker/dist/probe/index.js"],
|
||||
"ports": ["9876"],
|
||||
"state": "running",
|
||||
"env": { "MESH_NETWORK_CHECKER_PORT": "9876" }
|
||||
},
|
||||
{
|
||||
"id": "check",
|
||||
"type": "container",
|
||||
"name": "mesh-network-checker-check",
|
||||
"network": "host",
|
||||
"schedule": "*/5 * * * *",
|
||||
"args": ["run", "/app/modules/network-checker/dist/check/index.js"],
|
||||
"volumes": ["/var/lib/network-checker:/run/state"],
|
||||
"env": {
|
||||
"MESH_NETWORK_CHECKER_ROSTER": "/run/state/roster.json",
|
||||
"MESH_NETWORK_CHECKER_STATE": "/run/state",
|
||||
"MESH_NETWORK_CHECKER_PORT": "${port:9876}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
{
|
||||
"name": "@novox/module-network-checker",
|
||||
"version": "0.1.0",
|
||||
"description": "network-checker — dials what the mesh claims is reachable, from where the callers are, and says what it found.",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"devDependencies": { "@types/node": "^22.0.0", "typescript": "^5.6.0" }
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
// The endpoint the other machines' checkers dial (novox/hq ADR 0145).
|
||||
//
|
||||
// **This module's own endpoint is the instrument.** It is declared reachable over the private network
|
||||
// like any other service, so it is admitted by exactly the rule that governs every internally-exposed
|
||||
// service and it fails when that rule is wrong. A probe on a port that is never closed — ssh, say —
|
||||
// would have passed throughout the outage this module exists to catch.
|
||||
//
|
||||
// It accepts a connection and closes it. Answering anything would make this a protocol, and then the
|
||||
// question would be whether the protocol worked rather than whether the path did.
|
||||
|
||||
import { createServer } from "node:net";
|
||||
|
||||
const port = Number(process.env.MESH_NETWORK_CHECKER_PORT ?? "9876");
|
||||
|
||||
const server = createServer((socket) => {
|
||||
// Written before closing so a person dialling it by hand sees something, and so a half-open
|
||||
// connection is not mistaken for a working path by a client that only checks the handshake.
|
||||
socket.end("mesh network-checker\n");
|
||||
});
|
||||
|
||||
server.on("error", (err: Error) => {
|
||||
// Said and fatal: a probe that cannot listen must not look like a probe that nothing dialled.
|
||||
console.error(`network-checker: cannot serve the probe on ${port}: ${err.message}`);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
server.listen(port, () => console.log(`network-checker: probe listening on ${port}`));
|
||||
|
||||
for (const signal of ["SIGTERM", "SIGINT"] as const) {
|
||||
process.on(signal, () => server.close(() => process.exit(0)));
|
||||
}
|
||||
@@ -1,155 +0,0 @@
|
||||
// What the mesh claims is reachable, and how to find out (novox/hq ADR 0145).
|
||||
//
|
||||
// The mesh asserts three things are callable (ADR 0144): what runs on the same machine, another
|
||||
// machine's service exposed to the private network, and another machine's service exposed publicly.
|
||||
// This decides what to dial for each and reads the answers. It opens connections and nothing more —
|
||||
// the module that owns a service is the one that knows whether it is working.
|
||||
//
|
||||
// **The target is this module's own endpoint, and that is deliberate.** The obvious thing to dial is a
|
||||
// service every machine has, and the services every machine has are the ones never closed — ssh above
|
||||
// all. Dialling one of those would have passed throughout the outage this exists to catch, because what
|
||||
// broke was a service exposed to the private network and ssh is admitted unconditionally. A probe on a
|
||||
// port that cannot fail measures nothing.
|
||||
|
||||
import { connect } from "node:net";
|
||||
import { lookup } from "node:dns";
|
||||
|
||||
/** One machine as the mesh's roster describes it. */
|
||||
export interface Machine {
|
||||
name: string;
|
||||
fqdn: string;
|
||||
address: string;
|
||||
/** The name this machine is reached by from outside, where it has one. Absent for most machines, and
|
||||
* a machine with no public face has no public claim to check. */
|
||||
public?: string;
|
||||
}
|
||||
|
||||
/** The roster the mesh renders for this module: who this machine is, and who the others are. */
|
||||
export interface Roster {
|
||||
node: string;
|
||||
machines: Machine[];
|
||||
}
|
||||
|
||||
/** Which of the mesh's three claims a check is about, so a failure says which one broke. */
|
||||
export type Claim = "this machine" | "the private network" | "the public network";
|
||||
|
||||
/** One thing to dial. */
|
||||
export interface Target {
|
||||
claim: Claim;
|
||||
machine: string;
|
||||
/** What to dial — a name where the point is that names resolve, an address where it is not. */
|
||||
at: string;
|
||||
port: number;
|
||||
/** Whether `at` is a name that must resolve first, so a resolution failure is reported as one. */
|
||||
byName: boolean;
|
||||
}
|
||||
|
||||
/** What one dial found. */
|
||||
export interface Result extends Target {
|
||||
ok: boolean;
|
||||
/** Which step failed, so a reader is sent to the right place: the resolver, or the filter. */
|
||||
failed?: "resolution" | "connection";
|
||||
detail?: string;
|
||||
ms: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* targetsFor is everything this machine should be able to reach, from the roster it was given.
|
||||
*
|
||||
* Its own machine first, because that is the case that distinguishes a caller on the machine from a
|
||||
* caller in one of its containers — the one that broke. Then every other machine over the private
|
||||
* network. The public claim is only checked where a public address is known for a machine, because a
|
||||
* machine with no public face has nothing to fail.
|
||||
*/
|
||||
export function targetsFor(roster: Roster, probePort: number, publicPort?: number): Target[] {
|
||||
const out: Target[] = [];
|
||||
for (const m of roster.machines) {
|
||||
const own = m.name === roster.node;
|
||||
out.push({
|
||||
claim: own ? "this machine" : "the private network",
|
||||
machine: m.name,
|
||||
at: m.address,
|
||||
port: probePort,
|
||||
byName: false,
|
||||
});
|
||||
// And by name, because a name that does not resolve and a port that does not answer are different
|
||||
// faults with different owners.
|
||||
out.push({
|
||||
claim: own ? "this machine" : "the private network",
|
||||
machine: m.name,
|
||||
at: m.fqdn,
|
||||
port: probePort,
|
||||
byName: true,
|
||||
});
|
||||
}
|
||||
if (publicPort !== undefined) {
|
||||
for (const m of roster.machines) {
|
||||
if (!m.public) continue;
|
||||
out.push({
|
||||
claim: "the public network",
|
||||
machine: m.name,
|
||||
at: m.public,
|
||||
port: publicPort,
|
||||
byName: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** dial opens a connection and closes it. Whether the port accepts is the whole of what is asked. */
|
||||
export function dial(target: Target, timeoutMs: number): Promise<Result> {
|
||||
const began = Date.now();
|
||||
const done = (ok: boolean, failed?: Result["failed"], detail?: string): Result => ({
|
||||
...target, ok, failed, detail, ms: Date.now() - began,
|
||||
});
|
||||
|
||||
return new Promise<Result>((resolve) => {
|
||||
const open = () => {
|
||||
const socket = connect({ host: target.at, port: target.port });
|
||||
const finish = (r: Result) => { socket.destroy(); resolve(r); };
|
||||
socket.setTimeout(timeoutMs);
|
||||
socket.once("connect", () => finish(done(true)));
|
||||
socket.once("timeout", () => finish(done(false, "connection", "timed out")));
|
||||
socket.once("error", (err: Error) => finish(done(false, "connection", err.message)));
|
||||
};
|
||||
|
||||
if (!target.byName) { open(); return; }
|
||||
// Resolved first and reported separately: a checker that says "unreachable" for a name the
|
||||
// resolver never answered sends a reader to the filter, which is not where the fault is.
|
||||
lookup(target.at, (err) => {
|
||||
if (err) { resolve(done(false, "resolution", err.message)); return; }
|
||||
open();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/** A path's running count of consecutive failures, keyed so it survives between runs. */
|
||||
export type Counts = Record<string, number>;
|
||||
|
||||
/** keyOf names one path, stably, so a count follows it across runs. */
|
||||
export function keyOf(t: Target): string {
|
||||
return `${t.claim}|${t.machine}|${t.at}|${t.port}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* tally folds this run's results into the counts carried from the last one.
|
||||
*
|
||||
* **One failure is not a fault.** A machine rebooting is ordinary, and a checker that cries at the
|
||||
* first missed dial trains a reader to ignore it — which is worse than not checking (ADR 0145). A path
|
||||
* is broken once it has failed on consecutive runs, and the count travels with the result so a reader
|
||||
* can tell "briefly away" from "never worked".
|
||||
*/
|
||||
export function tally(results: Result[], before: Counts, threshold: number): {
|
||||
counts: Counts; broken: Array<Result & { consecutive: number }>;
|
||||
} {
|
||||
const counts: Counts = {};
|
||||
const broken: Array<Result & { consecutive: number }> = [];
|
||||
for (const r of results) {
|
||||
const key = keyOf(r);
|
||||
const n = r.ok ? 0 : (before[key] ?? 0) + 1;
|
||||
if (n > 0) counts[key] = n;
|
||||
if (n >= threshold) broken.push({ ...r, consecutive: n });
|
||||
}
|
||||
return { counts, broken };
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
import { strict as assert } from "node:assert";
|
||||
import test from "node:test";
|
||||
|
||||
import { keyOf, tally, targetsFor, type Result, type Roster } from "../reach.js";
|
||||
|
||||
const roster: Roster = {
|
||||
node: "here",
|
||||
machines: [
|
||||
{ name: "here", fqdn: "here.internal", address: "10.0.0.1" },
|
||||
{ name: "there", fqdn: "there.internal", address: "10.0.0.2", public: "there.example.test" },
|
||||
],
|
||||
};
|
||||
|
||||
test("its own machine is checked, which is the case that distinguishes a caller on it from one in a container", () => {
|
||||
const own = targetsFor(roster, 9876).filter((t) => t.claim === "this machine");
|
||||
assert.equal(own.length, 2, "its own machine by address and by name");
|
||||
assert.ok(own.some((t) => t.at === "10.0.0.1" && !t.byName));
|
||||
assert.ok(own.some((t) => t.at === "here.internal" && t.byName));
|
||||
});
|
||||
|
||||
test("every other machine is checked over the private network", () => {
|
||||
const other = targetsFor(roster, 9876).filter((t) => t.claim === "the private network");
|
||||
assert.deepEqual(other.map((t) => t.machine), ["there", "there"]);
|
||||
});
|
||||
|
||||
test("the public claim is only checked where a machine has a public name", () => {
|
||||
const pub = targetsFor(roster, 9876, 443).filter((t) => t.claim === "the public network");
|
||||
assert.equal(pub.length, 1, "only the machine with a public name");
|
||||
assert.equal(pub[0]!.at, "there.example.test");
|
||||
assert.equal(pub[0]!.port, 443);
|
||||
});
|
||||
|
||||
test("no public claim is made when no public port was given", () => {
|
||||
assert.equal(targetsFor(roster, 9876).filter((t) => t.claim === "the public network").length, 0);
|
||||
});
|
||||
|
||||
const failed = (at: string): Result => ({
|
||||
claim: "this machine", machine: "here", at, port: 9876, byName: false,
|
||||
ok: false, failed: "connection", ms: 1,
|
||||
});
|
||||
const passed = (at: string): Result => ({
|
||||
claim: "this machine", machine: "here", at, port: 9876, byName: false, ok: true, ms: 1,
|
||||
});
|
||||
|
||||
test("one failure is not a fault — a machine rebooting is ordinary", () => {
|
||||
const { counts, broken } = tally([failed("10.0.0.1")], {}, 2);
|
||||
assert.equal(broken.length, 0, "one missed dial says nothing");
|
||||
assert.equal(counts[keyOf(failed("10.0.0.1"))], 1, "and is remembered");
|
||||
});
|
||||
|
||||
test("a path that keeps failing is broken, and the count travels with it", () => {
|
||||
const first = tally([failed("10.0.0.1")], {}, 2);
|
||||
const second = tally([failed("10.0.0.1")], first.counts, 2);
|
||||
assert.equal(second.broken.length, 1);
|
||||
assert.equal(second.broken[0]!.consecutive, 2, "so a reader can tell briefly away from never worked");
|
||||
});
|
||||
|
||||
test("a path that recovers stops being counted", () => {
|
||||
const first = tally([failed("10.0.0.1")], {}, 2);
|
||||
const second = tally([passed("10.0.0.1")], first.counts, 2);
|
||||
assert.equal(second.broken.length, 0);
|
||||
assert.deepEqual(second.counts, {}, "nothing carried forward for a path that works");
|
||||
});
|
||||
|
||||
test("a count follows one path and not another", () => {
|
||||
const a = failed("10.0.0.1");
|
||||
const b = failed("10.0.0.2");
|
||||
const first = tally([a, b], {}, 2);
|
||||
const second = tally([a], first.counts, 2);
|
||||
assert.equal(second.broken.length, 1, "only the path dialled this run is judged");
|
||||
assert.equal(second.broken[0]!.at, "10.0.0.1");
|
||||
});
|
||||
@@ -1,12 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true
|
||||
},
|
||||
"include": ["reach.ts", "probe/index.ts", "check/index.ts", "test/*.ts"]
|
||||
}
|
||||
Reference in New Issue
Block a user