mesh-catalog/modules/network-checker/reach.ts

// What the mesh claims is reachable, and how to find out (novox/hq ADR 0145).
//
// The mesh asserts three things are callable (ADR 0144): what runs on the same machine, another
// machine's service exposed to the private network, and another machine's service exposed publicly.
// This decides what to dial for each and reads the answers. It opens connections and nothing more —
// the module that owns a service is the one that knows whether it is working.
//
// **The target is this module's own endpoint, and that is deliberate.** The obvious thing to dial is a
// service every machine has, and the services every machine has are the ones never closed — ssh above
// all. Dialling one of those would have passed throughout the outage this exists to catch, because what
// broke was a service exposed to the private network and ssh is admitted unconditionally. A probe on a
// port that cannot fail measures nothing.

import { connect } from "node:net";
import { lookup } from "node:dns";

/** One machine as the mesh's roster describes it. */
export interface Machine {
  name: string;
  fqdn: string;
  address: string;
  /** The name this machine is reached by from outside, where it has one. Absent for most machines, and
   *  a machine with no public face has no public claim to check. */
  public?: string;
}

/** The roster the mesh renders for this module: who this machine is, and who the others are. */
export interface Roster {
  node: string;
  machines: Machine[];
}

/** Which of the mesh's three claims a check is about, so a failure says which one broke. */
export type Claim = "this machine" | "the private network" | "the public network";

/** One thing to dial. */
export interface Target {
  claim: Claim;
  machine: string;
  /** What to dial — a name where the point is that names resolve, an address where it is not. */
  at: string;
  port: number;
  /** Whether `at` is a name that must resolve first, so a resolution failure is reported as one. */
  byName: boolean;
}

/** What one dial found. */
export interface Result extends Target {
  ok: boolean;
  /** Which step failed, so a reader is sent to the right place: the resolver, or the filter. */
  failed?: "resolution" | "connection";
  detail?: string;
  ms: number;
}

/**
 * targetsFor is everything this machine should be able to reach, from the roster it was given.
 *
 * Its own machine first, because that is the case that distinguishes a caller on the machine from a
 * caller in one of its containers — the one that broke. Then every other machine over the private
 * network. The public claim is only checked where a public address is known for a machine, because a
 * machine with no public face has nothing to fail.
 */
export function targetsFor(roster: Roster, probePort: number, publicPort?: number): Target[] {
  const out: Target[] = [];
  for (const m of roster.machines) {
    const own = m.name === roster.node;
    out.push({
      claim: own ? "this machine" : "the private network",
      machine: m.name,
      at: m.address,
      port: probePort,
      byName: false,
    });
    // And by name, because a name that does not resolve and a port that does not answer are different
    // faults with different owners.
    out.push({
      claim: own ? "this machine" : "the private network",
      machine: m.name,
      at: m.fqdn,
      port: probePort,
      byName: true,
    });
  }
  if (publicPort !== undefined) {
    for (const m of roster.machines) {
      if (!m.public) continue;
      out.push({
        claim: "the public network",
        machine: m.name,
        at: m.public,
        port: publicPort,
        byName: true,
      });
    }
  }
  return out;
}

/** dial opens a connection and closes it. Whether the port accepts is the whole of what is asked. */
export function dial(target: Target, timeoutMs: number): Promise<Result> {
  const began = Date.now();
  const done = (ok: boolean, failed?: Result["failed"], detail?: string): Result => ({
    ...target, ok, failed, detail, ms: Date.now() - began,
  });

  return new Promise<Result>((resolve) => {
    const open = () => {
      const socket = connect({ host: target.at, port: target.port });
      const finish = (r: Result) => { socket.destroy(); resolve(r); };
      socket.setTimeout(timeoutMs);
      socket.once("connect", () => finish(done(true)));
      socket.once("timeout", () => finish(done(false, "connection", "timed out")));
      socket.once("error", (err: Error) => finish(done(false, "connection", err.message)));
    };

    if (!target.byName) { open(); return; }
    // Resolved first and reported separately: a checker that says "unreachable" for a name the
    // resolver never answered sends a reader to the filter, which is not where the fault is.
    lookup(target.at, (err) => {
      if (err) { resolve(done(false, "resolution", err.message)); return; }
      open();
    });
  });
}

/** A path's running count of consecutive failures, keyed so it survives between runs. */
export type Counts = Record<string, number>;

/** keyOf names one path, stably, so a count follows it across runs. */
export function keyOf(t: Target): string {
  return `${t.claim}|${t.machine}|${t.at}|${t.port}`;
}

/**
 * tally folds this run's results into the counts carried from the last one.
 *
 * **One failure is not a fault.** A machine rebooting is ordinary, and a checker that cries at the
 * first missed dial trains a reader to ignore it — which is worse than not checking (ADR 0145). A path
 * is broken once it has failed on consecutive runs, and the count travels with the result so a reader
 * can tell "briefly away" from "never worked".
 */
export function tally(results: Result[], before: Counts, threshold: number): {
  counts: Counts; broken: Array<Result & { consecutive: number }>;
} {
  const counts: Counts = {};
  const broken: Array<Result & { consecutive: number }> = [];
  for (const r of results) {
    const key = keyOf(r);
    const n = r.ok ? 0 : (before[key] ?? 0) + 1;
    if (n > 0) counts[key] = n;
    if (n >= threshold) broken.push({ ...r, consecutive: n });
  }
  return { counts, broken };
}