jschoubben
784a5a6514
The mesh asserts three things are callable (ADR 0144) — what runs on the same machine, another machine's service exposed to the private network, and another machine's service exposed publicly — and has never checked any of them. The first was broken for eleven hours while the mesh reported every machine healthy. This runs on every machine, on the cadence the mesh already has, in its own container: the same position every other module calls from. Not the host and not the control plane, both of which reach these addresses by paths no ordinary caller uses and would have passed throughout that outage. **Its probe is its own endpoint, and that is the point.** Declared reachable over the private network like any other service, so it is admitted by exactly the rule that governs every internally-exposed service and fails when that rule is wrong. The tempting target is a service every machine has, and those are the ones never closed — ssh above all — which would have passed while the thing that actually broke was a service exposed to the private network. It resolves before it dials and says which failed, because a name that does not resolve and a port that does not answer have different owners. One failure is not a fault: a machine rebooting is ordinary, so a path is broken after consecutive runs and the count travels with the result. It reports and repairs nothing. novox/hq ADR 0145. Eight tests; the consecutive-failure logic proved by reverting it once. Not yet registered or assigned.
73 lines
3.1 KiB
TypeScript
73 lines
3.1 KiB
TypeScript
import { strict as assert } from "node:assert";
|
|
import test from "node:test";
|
|
|
|
import { keyOf, tally, targetsFor, type Result, type Roster } from "../reach.js";
|
|
|
|
const roster: Roster = {
|
|
node: "here",
|
|
machines: [
|
|
{ name: "here", fqdn: "here.internal", address: "10.0.0.1" },
|
|
{ name: "there", fqdn: "there.internal", address: "10.0.0.2", public: "there.example.test" },
|
|
],
|
|
};
|
|
|
|
test("its own machine is checked, which is the case that distinguishes a caller on it from one in a container", () => {
|
|
const own = targetsFor(roster, 9876).filter((t) => t.claim === "this machine");
|
|
assert.equal(own.length, 2, "its own machine by address and by name");
|
|
assert.ok(own.some((t) => t.at === "10.0.0.1" && !t.byName));
|
|
assert.ok(own.some((t) => t.at === "here.internal" && t.byName));
|
|
});
|
|
|
|
test("every other machine is checked over the private network", () => {
|
|
const other = targetsFor(roster, 9876).filter((t) => t.claim === "the private network");
|
|
assert.deepEqual(other.map((t) => t.machine), ["there", "there"]);
|
|
});
|
|
|
|
test("the public claim is only checked where a machine has a public name", () => {
|
|
const pub = targetsFor(roster, 9876, 443).filter((t) => t.claim === "the public network");
|
|
assert.equal(pub.length, 1, "only the machine with a public name");
|
|
assert.equal(pub[0]!.at, "there.example.test");
|
|
assert.equal(pub[0]!.port, 443);
|
|
});
|
|
|
|
test("no public claim is made when no public port was given", () => {
|
|
assert.equal(targetsFor(roster, 9876).filter((t) => t.claim === "the public network").length, 0);
|
|
});
|
|
|
|
const failed = (at: string): Result => ({
|
|
claim: "this machine", machine: "here", at, port: 9876, byName: false,
|
|
ok: false, failed: "connection", ms: 1,
|
|
});
|
|
const passed = (at: string): Result => ({
|
|
claim: "this machine", machine: "here", at, port: 9876, byName: false, ok: true, ms: 1,
|
|
});
|
|
|
|
test("one failure is not a fault — a machine rebooting is ordinary", () => {
|
|
const { counts, broken } = tally([failed("10.0.0.1")], {}, 2);
|
|
assert.equal(broken.length, 0, "one missed dial says nothing");
|
|
assert.equal(counts[keyOf(failed("10.0.0.1"))], 1, "and is remembered");
|
|
});
|
|
|
|
test("a path that keeps failing is broken, and the count travels with it", () => {
|
|
const first = tally([failed("10.0.0.1")], {}, 2);
|
|
const second = tally([failed("10.0.0.1")], first.counts, 2);
|
|
assert.equal(second.broken.length, 1);
|
|
assert.equal(second.broken[0]!.consecutive, 2, "so a reader can tell briefly away from never worked");
|
|
});
|
|
|
|
test("a path that recovers stops being counted", () => {
|
|
const first = tally([failed("10.0.0.1")], {}, 2);
|
|
const second = tally([passed("10.0.0.1")], first.counts, 2);
|
|
assert.equal(second.broken.length, 0);
|
|
assert.deepEqual(second.counts, {}, "nothing carried forward for a path that works");
|
|
});
|
|
|
|
test("a count follows one path and not another", () => {
|
|
const a = failed("10.0.0.1");
|
|
const b = failed("10.0.0.2");
|
|
const first = tally([a, b], {}, 2);
|
|
const second = tally([a], first.counts, 2);
|
|
assert.equal(second.broken.length, 1, "only the path dialled this run is judged");
|
|
assert.equal(second.broken[0]!.at, "10.0.0.1");
|
|
});
|