Raise a machine's network from what its engine says, once (hq ADR 0241)

A VPN client rewrote the laptop's resolver file and nothing said so. The
engine now states its machine's networking; the controller keeps it with
the machine's health (migration 0077) and raises the rewrite as its own
finding naming the writer, the machine's own faults as machine.<m>.network,
and what several machines cannot reach once, there. The gate waits on a
rewrite it did not make rather than putting back a good build.

cmd/mesh-controller/gate.go

@@ -283,6 +283,8 @@ type machineWord struct {
 	facts gateFacts
 	on    map[string]string
 	whole string
+	// waiting is what holds the machine on something shown to be another's (ADR 0241): the judging waits.
+	waiting string
 }
 
 // kindCoreBehind is D10's kind: a machine runs core components older than the mesh holds, or has not
@@ -346,6 +348,13 @@ func aboutTheMachine(machine string, moved []string, since time.Time, f gateFact
 			}
 		case coreBehind:
 			// Not what the send moved: said nowhere against it.
+		case c.Kind == kindNetworkRewritten || c.Kind == kindNetworkUnreachable && c.Subject.ID != machine:
+			// **Shown to be somebody else's** (ADR 0241): another program rewrote the resolver file the send
+			// did not move, or the machine cannot reach another that is down. Nothing the send did; the
+			// judging waits for it rather than putting back a build at the bound.
+			if w.waiting == "" {
+				w.waiting = machine + "'s network: " + said
+			}
 		default:
 			if w.whole == "" {
 				w.whole = machine + " as a whole: " + said
@@ -433,6 +442,8 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 				h, said = healthNotYet, on
 			} else if w.whole != "" {
 				h, said = healthNotYet, w.whole
+			} else if w.waiting != "" {
+				h, said = healthWaiting, w.waiting
 			}
 		}
 		if h != healthGood && !slices.Contains(failing, j.module) {

cmd/mesh-controller/machine_network.go

@@ -0,0 +1,359 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A machine says how its network is (novox/hq ADR 0241, which extends ADR 0240 from what a module runs
+// to the machine it runs on).
+//
+// **Every machine's node-engine judges its own networking** — the resolver file the uplink holder
+// declared, the names through every resolver it lists, the tunnel's handshake with the hub, the bus, the
+// default route — on the two-look rule, and states it beside its resources. The controller keeps the
+// newest statement per machine and raises from all of them together:
+//
+//   - **an outside writer of the resolver file is its own finding**, `machine.<m>.<owner>.rewritten`:
+//     the file the uplink holder declares was rewritten by another program, named where the engine could
+//     name it. The names failing through what that program wrote are that finding's consequence, said in
+//     it — never a second condition;
+//   - **what is the machine's own** — its route, its tunnel, its resolvers answering wrong, a resolver
+//     that is no mesh machine — is `machine.<m>.network`;
+//   - **what points at another machine is said once, there** (the provider hold of ADR 0240 rule 5, for
+//     the network): a failure toward the hub or toward a mesh resolver is held under that machine when it
+//     is down on the record — silent, or its own network unhealthy — or when a second machine finds the
+//     same; then `machine.<x>.unreachable` names every machine that cannot reach it, and none of them
+//     raises anything of its own for it. One machine alone failing toward a healthy one is its own.
+//
+// Each is a warning; urgent on the control node, or when the bus cannot be reached, or for the hub.
+// Cleared on the first statement that no longer says it. An engine older than this judging says nothing
+// of its network, and nothing is raised for it.
+
+// The conditions a machine's network raises.
+const (
+	kindMachineNetwork     = "machine-network"
+	kindNetworkRewritten   = "network-rewritten"
+	kindNetworkUnreachable = "network-unreachable"
+	sourceNetwork          = "network"
+)
+
+// networkKinds are the kinds this judging owns: every open one it no longer says, it clears.
+var networkKinds = []string{kindMachineNetwork, kindNetworkRewritten, kindNetworkUnreachable}
+
+// networkFacts is what one judging of every machine's network reads.
+type networkFacts struct {
+	healths map[string]inventory.NodeHealth
+	// byAddress is each machine's address on the private network; hub the hub's name; control the
+	// control node's.
+	byAddress map[string]string
+	hub       string
+	control   string
+	// silent is every machine whose silence is an open condition.
+	silent map[string]bool
+}
+
+// judgeNetworks raises and clears every machine's network conditions from every machine's newest
+// statement, after one machine's statement was kept.
+func judgeNetworks(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, now time.Time) error {
+	healths, err := inv.Healths(ctx)
+	if err != nil {
+		return err
+	}
+	overlays, err := inv.Overlays(ctx)
+	if err != nil {
+		return err
+	}
+	open, err := k.Open(ctx)
+	if err != nil {
+		return err
+	}
+	f := networkFacts{healths: healths, byAddress: map[string]string{}, control: controlHost(ctx, inv),
+		silent: map[string]bool{}}
+	for _, o := range overlays {
+		if o.Address != "" {
+			f.byAddress[o.Address] = o.Name
+		}
+		if o.Hub {
+			f.hub = o.Name
+		}
+	}
+	for _, c := range open {
+		if c.Subject.Scope == conditions.ScopeMachine && c.Kind == "silent" {
+			f.silent[c.Subject.ID] = true
+		}
+	}
+	var problems []string
+	said := map[string]bool{}
+	for _, o := range networkObservations(f) {
+		said[o.Key()] = true
+		if _, err := k.Observe(ctx, o); err != nil {
+			problems = append(problems, err.Error())
+		}
+	}
+	for _, c := range open {
+		if !slices.Contains(networkKinds, c.Kind) || said[c.Key] {
+			continue
+		}
+		why := "no machine says it any more"
+		if c.Subject.Machine != "" {
+			why = c.Subject.Machine + "'s network no longer says it"
+		}
+		if _, err := k.Clear(ctx, c.Key, why); err != nil {
+			problems = append(problems, err.Error())
+		}
+	}
+	if len(problems) > 0 {
+		return fmt.Errorf("%s", strings.Join(problems, "; "))
+	}
+	return nil
+}
+
+// pointed is one machine's failing part that points at another machine.
+type pointed struct {
+	from string
+	part inventory.NetworkPart
+}
+
+// networkObservations is every network condition the statements say now. Pure.
+func networkObservations(f networkFacts) []conditions.Observation {
+	machines := make([]string, 0, len(f.healths))
+	for m := range f.healths {
+		machines = append(machines, m)
+	}
+	sort.Strings(machines)
+
+	unhealthy := func(m string) []inventory.NetworkPart {
+		h := f.healths[m]
+		if h.Network == nil {
+			return nil
+		}
+		var out []inventory.NetworkPart
+		for _, p := range h.Network.Parts {
+			if p.State == link.StateUnhealthy {
+				out = append(out, p)
+			}
+		}
+		return out
+	}
+	// The machines a part points at, other than its own: the hub, and each mesh resolver by its address.
+	// An address that is no mesh machine's — the resolver a VPN client wrote in — is the machine's own.
+	targets := func(m string, p inventory.NetworkPart) ([]string, bool) {
+		if len(p.Toward) == 0 {
+			return nil, false
+		}
+		var out []string
+		for _, t := range p.Toward {
+			x := f.byAddress[t]
+			if t == link.TowardHub {
+				x = f.hub
+			}
+			if x == "" || x == m {
+				return nil, false
+			}
+			if !slices.Contains(out, x) {
+				out = append(out, x)
+			}
+		}
+		return out, true
+	}
+
+	// First pass: what points at whom.
+	pointing := map[string][]pointed{}
+	for _, m := range machines {
+		for _, p := range unhealthy(m) {
+			if xs, ok := targets(m, p); ok {
+				for _, x := range xs {
+					pointing[x] = append(pointing[x], pointed{from: m, part: p})
+				}
+			}
+		}
+	}
+	from := func(x string) []string {
+		var out []string
+		for _, pt := range pointing[x] {
+			if !slices.Contains(out, pt.from) {
+				out = append(out, pt.from)
+			}
+		}
+		sort.Strings(out)
+		return out
+	}
+	// A machine is down on the record when its silence is open or its own network is unhealthy, or when
+	// two machines find it unreachable: then what points at it is held there.
+	down := func(x string) bool {
+		return f.silent[x] || len(unhealthy(x)) > 0 || len(from(x)) >= 2
+	}
+
+	var out []conditions.Observation
+	saysOwn := map[string]bool{}
+	for _, m := range machines {
+		parts := unhealthy(m)
+		if len(parts) == 0 {
+			continue
+		}
+		var own []inventory.NetworkPart
+		var rewritten *inventory.NetworkPart
+		for i, p := range parts {
+			if p.Part == link.PartResolvConf {
+				rewritten = &parts[i]
+				continue
+			}
+			if xs, ok := targets(m, p); ok && allDown(xs, down) {
+				continue // held at the machines it points at
+			}
+			own = append(own, p)
+		}
+		if rewritten != nil {
+			out = append(out, rewrittenObservation(m, *rewritten, parts, f))
+			// The names failing through what another program wrote are that finding's, said in it.
+			kept := own[:0]
+			for _, p := range own {
+				if p.Part != link.PartNames {
+					kept = append(kept, p)
+				}
+			}
+			own = kept
+		}
+		if len(own) > 0 {
+			out = append(out, machineNetworkObservation(m, own, f, from(m)))
+			saysOwn[m] = true
+		}
+	}
+	// Said once, at the machine everybody points at — unless its own network condition already says it
+	// (listed there), or its silence does.
+	targetsSorted := make([]string, 0, len(pointing))
+	for x := range pointing {
+		targetsSorted = append(targetsSorted, x)
+	}
+	sort.Strings(targetsSorted)
+	for _, x := range targetsSorted {
+		if !down(x) || saysOwn[x] || f.silent[x] {
+			continue
+		}
+		out = append(out, unreachableObservation(x, pointing[x], from(x), f))
+	}
+	return out
+}
+
+func allDown(xs []string, down func(string) bool) bool {
+	for _, x := range xs {
+		if !down(x) {
+			return false
+		}
+	}
+	return len(xs) > 0
+}
+
+// rewrittenObservation is the resolver file rewritten by another program: its own finding, naming the
+// writer where the engine could, and what it costs the machine.
+func rewrittenObservation(m string, p inventory.NetworkPart, all []inventory.NetworkPart, f networkFacts) conditions.Observation {
+	writer := ""
+	if p.Writer != "" {
+		writer = " (" + p.Writer + ")"
+	}
+	cost := "the names through it are not yet judged"
+	said := []string{p.Part + ": " + p.Said}
+	for _, q := range all {
+		if q.Part == link.PartNames {
+			cost = strings.TrimSuffix(q.Reason, ".")
+			said = append(said, q.Part+": "+q.Said)
+		}
+	}
+	if cost == "the names through it are not yet judged" {
+		cost = "the names still resolve through what it wrote"
+	}
+	id := m
+	if p.Owner != "" {
+		// Named by the module whose file it is: a send that moved that module is what the gate
+		// holds it on (issue 281's rule — what names a moved module is that module's).
+		id = m + "." + p.Owner
+	}
+	severity := conditions.Warning
+	if m == f.control {
+		severity = conditions.Urgent
+	}
+	summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the "+
+		"node-engine writes it back at its next reconcile, or that program gives it back", m, writer, cost)
+	if p.Owner != "" {
+		summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until "+
+			"the node-engine writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost)
+	}
+	return conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "rewritten", Kind: kindNetworkRewritten,
+		Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary,
+		Said: fmt.Sprintf("since %s: %s", p.Since.UTC().Format("2006-01-02 15:04:05 MST"), strings.Join(said, " | "))}
+}
+
+// machineNetworkObservation is what is wrong with a machine's own networking.
+func machineNetworkObservation(m string, parts []inventory.NetworkPart, f networkFacts, waiting []string) conditions.Observation {
+	var words, said []string
+	severity := conditions.Warning
+	for _, p := range parts {
+		words = append(words, p.Reason)
+		said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, p.Since.UTC().Format("2006-01-02 15:04:05 MST"), p.Said))
+		if p.Part == link.PartBus {
+			severity = conditions.Urgent
+		}
+	}
+	if m == f.control || m == f.hub {
+		severity = conditions.Urgent
+	}
+	summary := fmt.Sprintf("%s's network is not healthy: %s", m, strings.Join(words, "; "))
+	if len(waiting) > 0 {
+		severity = conditions.Urgent
+		summary += fmt.Sprintf("; %s cannot reach it", strings.Join(waiting, ", "))
+	}
+	return conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Token: "network", Kind: kindMachineNetwork,
+		Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary, Said: strings.Join(said, " | ")}
+}
+
+// unreachableObservation is one machine others cannot reach, said once there.
+func unreachableObservation(x string, pts []pointed, from []string, f networkFacts) conditions.Observation {
+	var what []string
+	var said []string
+	for _, pt := range pts {
+		w := map[string]string{link.PartTunnel: "the tunnel to it", link.PartBus: "the bus on it",
+			link.PartNames: "its resolver"}[pt.part.Part]
+		if w == "" {
+			w = pt.part.Part
+		}
+		if !slices.Contains(what, w) {
+			what = append(what, w)
+		}
+		said = append(said, fmt.Sprintf("%s: %s: %s", pt.from, pt.part.Part, pt.part.Said))
+	}
+	severity := conditions.Warning
+	if x == f.hub || x == f.control || slices.Contains(what, "the bus on it") {
+		severity = conditions.Urgent
+	}
+	return conditions.Observation{Scope: conditions.ScopeMachine, ID: x, Token: "unreachable", Kind: kindNetworkUnreachable,
+		Machine: x, Also: from, Severity: severity, Source: sourceNetwork,
+		Summary: fmt.Sprintf("%s cannot be reached from %s: %s", x, strings.Join(from, ", "), strings.Join(what, ", ")),
+		Said:    strings.Join(said, " | ")}
+}
+
+// networkLines is what `node show` says of a machine's networking.
+func networkLines(h inventory.NodeHealth, had bool, now time.Time) []string {
+	if !had || h.Network == nil {
+		return []string{"  its node-engine does not say how its network is — it is older than that judging (ADR 0241)"}
+	}
+	out := []string{fmt.Sprintf("  its network: %s since %s", h.Network.State, h.Network.Since.Local().Format("2006-01-02 15:04"))}
+	for _, p := range h.Network.Parts {
+		line := fmt.Sprintf("    %-10s %s", p.State, p.Part)
+		if p.Reason != "" {
+			line += " — " + p.Reason
+		}
+		if p.Writer != "" {
+			line += " (" + p.Writer + ")"
+		}
+		out = append(out, line)
+	}
+	return out
+}

cmd/mesh-controller/machine_network_test.go

@@ -0,0 +1,306 @@
+package main
+
+import (
+	"encoding/json"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A machine says how its network is (novox/hq ADR 0241, "how it is checked"): the resolver file rewritten
+// by another program is one finding, naming the writer, with the names it costs said in it; what is the
+// machine's own is `machine.<m>.network`; what points at another machine that is down is said once, there;
+// one machine alone failing toward a healthy one is its own; the control node, the hub and the bus are
+// urgent; an engine that says nothing of its network raises nothing; and the gate waits on what is shown to
+// be another's.
+
+func aNetwork(state string, parts ...inventory.NetworkPart) *inventory.NetworkHealth {
+	for i := range parts {
+		if parts[i].State == "" {
+			parts[i].State = link.StateUnhealthy
+		}
+		parts[i].Since = h0
+	}
+	return &inventory.NetworkHealth{State: state, Since: h0, Parts: parts}
+}
+
+func netFacts(healths map[string]*inventory.NetworkHealth) networkFacts {
+	f := networkFacts{healths: map[string]inventory.NodeHealth{}, hub: "anchor", control: "anchor",
+		byAddress: map[string]string{"10.77.0.1": "anchor", "10.77.0.2": "laptop", "10.77.0.3": "spare"},
+		silent:    map[string]bool{}}
+	for m, n := range healths {
+		f.healths[m] = inventory.NodeHealth{Node: m, Network: n}
+	}
+	return f
+}
+
+var (
+	rewrittenByVPN = inventory.NetworkPart{Part: link.PartResolvConf, Reason: "the resolver file was rewritten by another program",
+		Said: "/etc/resolv.conf lists 172.16.5.5 where 10.77.0.1 is declared", Writer: "FortiClient", Owner: "networkmanager"}
+	namesThroughVPN = inventory.NetworkPart{Part: link.PartNames, Reason: "mesh names do not resolve",
+		Said: "172.16.5.5 — anchor.internal (IPv4): says no such name", Toward: []string{"172.16.5.5"}}
+	tunnelDown = inventory.NetworkPart{Part: link.PartTunnel, Reason: "the tunnel to the hub has not handshaken for over five minutes",
+		Said: "mesh0's newest handshake with the hub was 9m0s ago", Toward: []string{link.TowardHub}}
+	noRoute = inventory.NetworkPart{Part: link.PartRoute, Reason: "the machine has no default route", Said: "no default route"}
+)
+
+func keysOf(obs []conditions.Observation) []string {
+	var keys []string
+	for _, o := range obs {
+		keys = append(keys, o.Key())
+	}
+	return keys
+}
+
+func TestAResolverFileRewrittenIsOneFindingNamingItsWriterAndWhatItCosts(t *testing.T) {
+	obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
+		"anchor": aNetwork(link.StateHealthy),
+		"laptop": aNetwork(link.StateUnhealthy, rewrittenByVPN, namesThroughVPN),
+	}))
+	if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" {
+		t.Fatalf("want one finding, the rewrite, got %v", keys)
+	}
+	o := obs[0]
+	for _, want := range []string{"laptop", "rewritten by another program (FortiClient)", "mesh names do not resolve",
+		"next reconcile"} {
+		if !strings.Contains(o.Summary, want) {
+			t.Errorf("the summary does not say %q: %s", want, o.Summary)
+		}
+	}
+	if strings.Contains(o.Summary, "172.16.") || strings.Contains(o.Summary, "/etc/") {
+		t.Errorf("an address or a path reached the summary: %s", o.Summary)
+	}
+	if !strings.Contains(o.Said, "172.16.5.5") || o.Severity != conditions.Warning || o.Machine != "laptop" {
+		t.Errorf("the evidence or the severity is wrong: %+v", o)
+	}
+}
+
+func TestOneMachineFailingTowardAHealthyHubIsItsOwn(t *testing.T) {
+	obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
+		"anchor": aNetwork(link.StateHealthy),
+		"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
+		"spare":  aNetwork(link.StateHealthy),
+	}))
+	if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" {
+		t.Fatalf("want the laptop's own, got %v", keys)
+	}
+	if obs[0].Severity != conditions.Warning {
+		t.Fatalf("a laptop's own tunnel is a warning, got %s", obs[0].Severity)
+	}
+}
+
+func TestTwoMachinesThatCannotReachTheHubAreSaidOnceAtTheHub(t *testing.T) {
+	obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
+		"anchor": aNetwork(link.StateHealthy),
+		"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
+		"spare":  aNetwork(link.StateUnhealthy, tunnelDown),
+	}))
+	if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.unreachable" {
+		t.Fatalf("want one condition at the hub, got %v", keys)
+	}
+	o := obs[0]
+	if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop, spare") || len(o.Also) != 2 {
+		t.Fatalf("the hub's condition is %+v", o)
+	}
+}
+
+func TestWhatPointsAtASilentHubIsHeldUnderItsSilence(t *testing.T) {
+	f := netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, tunnelDown)})
+	f.silent["anchor"] = true
+	if obs := networkObservations(f); len(obs) != 0 {
+		t.Fatalf("the hub's silence says it; got %v", keysOf(obs))
+	}
+}
+
+func TestAHubWhoseOwnNetworkIsUnhealthyListsWhoCannotReachIt(t *testing.T) {
+	obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
+		"anchor": aNetwork(link.StateUnhealthy, noRoute),
+		"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
+	}))
+	if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.network" {
+		t.Fatalf("want the hub's own, holding the laptop's, got %v", keys)
+	}
+	if o := obs[0]; o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop cannot reach it") {
+		t.Fatalf("the hub's condition is %+v", o)
+	}
+}
+
+func TestOneMachineFailingAHealthyMeshResolverIsItsOwnAndTwoAreTheResolvers(t *testing.T) {
+	silentResolver := inventory.NetworkPart{Part: link.PartNames, Reason: "1 of its 2 resolvers do not answer as the mesh's do",
+		Said: "10.77.0.3 — no answer within 1s", Toward: []string{"10.77.0.3"}}
+	obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
+		"anchor": aNetwork(link.StateHealthy), "spare": aNetwork(link.StateHealthy),
+		"laptop": aNetwork(link.StateUnhealthy, silentResolver),
+	}))
+	if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" {
+		t.Fatalf("one machine alone: want its own, got %v", keys)
+	}
+	obs = networkObservations(netFacts(map[string]*inventory.NetworkHealth{
+		"anchor": aNetwork(link.StateUnhealthy, silentResolver), "spare": aNetwork(link.StateHealthy),
+		"laptop": aNetwork(link.StateUnhealthy, silentResolver),
+	}))
+	if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.spare.unreachable" {
+		t.Fatalf("two machines: want it said once at the resolver's machine, got %v", keys)
+	}
+	if !strings.Contains(obs[0].Summary, "its resolver") {
+		t.Fatalf("the resolver's machine is said %s", obs[0].Summary)
+	}
+}
+
+func TestTheControlNodeAndTheBusAreUrgent(t *testing.T) {
+	f := netFacts(map[string]*inventory.NetworkHealth{"anchor": aNetwork(link.StateUnhealthy, rewrittenByVPN)})
+	if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent {
+		t.Fatalf("the control node's rewritten file: %+v", obs)
+	}
+	bus := inventory.NetworkPart{Part: link.PartBus, Reason: "the bus cannot be reached", Said: "no link"}
+	f = netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, bus, noRoute)})
+	if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent {
+		t.Fatalf("the bus unreachable from the laptop: %+v", obs)
+	}
+}
+
+func TestAnEngineThatSaysNothingOfItsNetworkRaisesNothing(t *testing.T) {
+	f := netFacts(map[string]*inventory.NetworkHealth{"laptop": nil, "anchor": aNetwork(link.StateHealthy)})
+	if obs := networkObservations(f); len(obs) != 0 {
+		t.Fatalf("got %v", keysOf(obs))
+	}
+}
+
+// Through the store and the keeper: the statement kept, the rewrite raised from it, cleared when the file
+// is written back, and node show saying it.
+func TestARewrittenResolverFileIsRaisedFromTheStatementAndClearedWhenWrittenBack(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv, k := open.inventory, conditionsFrom
+	say := func(at time.Time, n *link.NetworkHealth) {
+		t.Helper()
+		if err := stateHealth(ctx, inv, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil {
+			t.Fatal(err)
+		}
+	}
+	openKeys := func() []string {
+		t.Helper()
+		list, err := k.Open(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var keys []string
+		for _, c := range list {
+			keys = append(keys, c.Key)
+		}
+		return keys
+	}
+	healthy := &link.NetworkHealth{State: link.StateHealthy, Since: h0, Parts: []link.NetworkPart{
+		{Part: link.PartResolvConf, State: link.StateHealthy, Since: h0}}}
+	rewritten := &link.NetworkHealth{State: link.StateUnhealthy, Since: h0.Add(time.Minute), Parts: []link.NetworkPart{
+		{Part: link.PartResolvConf, State: link.StateUnhealthy, Reason: rewrittenByVPN.Reason, Said: rewrittenByVPN.Said,
+			Writer: "FortiClient", Owner: "networkmanager", Since: h0.Add(time.Minute)},
+		{Part: link.PartNames, State: link.StateUnhealthy, Reason: namesThroughVPN.Reason, Said: namesThroughVPN.Said,
+			Toward: namesThroughVPN.Toward, Since: h0.Add(time.Minute)}}}
+
+	say(h0, healthy)
+	if keys := openKeys(); len(keys) != 0 {
+		t.Fatalf("a healthy network raised %v", keys)
+	}
+	say(h0.Add(time.Minute), rewritten)
+	if keys := openKeys(); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" {
+		t.Fatalf("the rewrite raised %v", keys)
+	}
+	kept, had, err := inv.HealthOf(ctx, "laptop")
+	if err != nil || !had || kept.Network == nil || kept.Network.Parts[0].Writer != "FortiClient" {
+		t.Fatalf("the statement's network was not kept: %+v %v", kept.Network, err)
+	}
+	if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "FortiClient") ||
+		!strings.Contains(lines, "unhealthy  resolv-conf") {
+		t.Fatalf("node show says:\n%s", lines)
+	}
+	say(h0.Add(2*time.Minute), healthy)
+	if keys := openKeys(); len(keys) != 0 {
+		t.Fatalf("written back, still open: %v", keys)
+	}
+	// An engine older than the judging says no network: nothing raised, and node show says it is not known.
+	say(h0.Add(3*time.Minute), nil)
+	kept, had, _ = inv.HealthOf(ctx, "laptop")
+	if keys := openKeys(); len(keys) != 0 || kept.Network != nil {
+		t.Fatalf("an older engine: %v %+v", keys, kept.Network)
+	}
+	if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "older than that judging") {
+		t.Fatalf("node show says:\n%s", lines)
+	}
+}
+
+// The gate: a resolver file another program rewrote waits the judging rather than failing it at the
+// bound; the same, when the send moved the module whose file it is, is that module's; a machine's own
+// network fault holds the machine as a whole, as before.
+func TestTheGateWaitsOnARewriteItDidNotMakeAndHoldsTheOwnerOnOneItMoved(t *testing.T) {
+	since := h0
+	rewrite := conditions.Condition{Key: "machine.laptop.networkmanager.rewritten", Kind: kindNetworkRewritten,
+		Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop.networkmanager", Machine: "laptop"},
+		Summary: "the resolver file was rewritten", Raised: since.Add(time.Minute), Source: sourceNetwork}
+	f := gateFacts{judged: true, open: []conditions.Condition{rewrite}}
+	if w := aboutTheMachine("laptop", []string{"letta"}, since, f); w.waiting == "" || w.whole != "" || len(w.on) != 0 {
+		t.Fatalf("a rewrite the send did not make: %+v", w)
+	}
+	if w := aboutTheMachine("laptop", []string{"networkmanager", "letta"}, since, f); w.on["networkmanager"] == "" ||
+		w.on["letta"] != "" || w.waiting != "" {
+		t.Fatalf("a rewrite of the file a moved module owns: %+v", w)
+	}
+	own := conditions.Condition{Key: "machine.laptop.network", Kind: kindMachineNetwork,
+		Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"},
+		Summary: "laptop's network is not healthy", Raised: since.Add(time.Minute), Source: sourceNetwork}
+	if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true,
+		open: []conditions.Condition{own}}); w.whole == "" || w.waiting != "" {
+		t.Fatalf("the machine's own network: %+v", w)
+	}
+	unreachable := conditions.Condition{Key: "machine.anchor.unreachable", Kind: kindNetworkUnreachable,
+		Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor", Also: []string{"laptop", "spare"}},
+		Summary: "anchor cannot be reached", Raised: since.Add(time.Minute), Source: sourceNetwork}
+	if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true,
+		open: []conditions.Condition{unreachable}}); w.waiting == "" || w.whole != "" {
+		t.Fatalf("a machine that cannot reach the hub: %+v", w)
+	}
+}
+
+// TestTheDrillsStatementsRaiseAndClearTheRewrite replays the drill of ADR 0241 (mesh-host
+// internal/network TestDrill…): what a node-engine said in a throwaway container while its resolver file
+// was declared, rewritten as a VPN client rewrites it, and written back — recorded, with the mesh's names
+// and addresses replaced by this test mesh's. Healthy raises nothing; the rewrite, on its second look, is
+// raised as one finding naming the writer; written back, it clears.
+func TestTheDrillsStatementsRaiseAndClearTheRewrite(t *testing.T) {
+	raw, err := os.ReadFile("testdata/network-drill.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var said []link.Health
+	if err := json.Unmarshal(raw, &said); err != nil {
+		t.Fatal(err)
+	}
+	open := aMesh(t)
+	ctx := t.Context()
+	k := conditionsFrom
+	var raisedAt []int
+	for i, h := range said {
+		if err := stateHealth(ctx, open.inventory, k, "laptop", h, h.At); err != nil {
+			t.Fatal(err)
+		}
+		list, err := k.Open(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, c := range list {
+			if c.Key != "machine.laptop.networkmanager.rewritten" || !strings.Contains(c.Summary, "(FortiClient)") {
+				t.Fatalf("statement %d raised %s: %s", i, c.Key, c.Summary)
+			}
+			raisedAt = append(raisedAt, i)
+		}
+	}
+	// Five statements: healthy, healthy, one failing look (still healthy), unhealthy, written back.
+	if len(raisedAt) != 1 || raisedAt[0] != 3 {
+		t.Fatalf("the rewrite was open after statements %v; want after the fourth alone", raisedAt)
+	}
+}

cmd/mesh-controller/module_health.go

@@ -83,8 +83,16 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
 	for module := range unhealthy {
 		streaks[module] = prev.Streaks[module] + 1
 	}
+	var network *inventory.NetworkHealth
+	if h.Network != nil {
+		network = &inventory.NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: []inventory.NetworkPart{}}
+		for _, p := range h.Network.Parts {
+			network.Parts = append(network.Parts, inventory.NetworkPart{Part: p.Part, State: p.State, Reason: p.Reason,
+				Said: p.Said, Writer: p.Writer, Owner: p.Owner, Toward: p.Toward, Since: p.Since, Streak: p.Streak})
+		}
+	}
 	stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At,
-		HeardAt: now, Resources: resources, Streaks: streaks})
+		HeardAt: now, Resources: resources, Streaks: streaks, Network: network})
 	if err != nil || !stored {
 		if err == nil {
 			healthRefused.Add(1)
@@ -94,7 +102,16 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
 	if k == nil {
 		return nil
 	}
-	return judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now)
+	err = judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now)
+	// And every machine's network, from every machine's newest statement (ADR 0241): a statement about one
+	// machine can hold another's finding, or release it.
+	if nerr := judgeNetworks(ctx, inv, k, now); nerr != nil {
+		if err == nil {
+			return nerr
+		}
+		return fmt.Errorf("%w; %v", err, nerr)
+	}
+	return err
 }
 
 // judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a

cmd/mesh-controller/nodes.go

@@ -546,6 +546,9 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
 		for _, line := range healthLines(h, had, time.Now()) {
 			fmt.Println(line)
 		}
+		for _, line := range networkLines(h, had, time.Now()) {
+			fmt.Println(line)
+		}
 	}
 
 	held, err := inv.Profile(ctx, name)

cmd/mesh-controller/testdata/network-drill.json

@@ -0,0 +1,170 @@
+[
+  {
+    "contract": 2,
+    "at": "2026-10-07T16:48:58.691388404Z",
+    "resources": [],
+    "network": {
+      "state": "healthy",
+      "since": "2026-10-07T16:48:58.691388404Z",
+      "parts": [
+        {
+          "part": "resolv-conf",
+          "state": "healthy",
+          "since": "2026-10-07T16:48:58.691388404Z"
+        },
+        {
+          "part": "names",
+          "state": "healthy",
+          "since": "2026-10-07T16:48:58.691388404Z"
+        },
+        {
+          "part": "bus",
+          "state": "healthy",
+          "since": "2026-10-07T16:48:58.691388404Z"
+        },
+        {
+          "part": "route",
+          "state": "healthy",
+          "since": "2026-10-07T16:48:58.691388404Z"
+        }
+      ]
+    }
+  },
+  {
+    "contract": 2,
+    "at": "2026-10-07T16:49:28.691388404Z",
+    "resources": [],
+    "network": {
+      "state": "healthy",
+      "since": "2026-10-07T16:48:58.691388404Z",
+      "parts": [
+        {
+          "part": "resolv-conf",
+          "state": "healthy",
+          "since": "2026-10-07T16:48:58.691388404Z"
+        },
+        {
+          "part": "names",
+          "state": "healthy",
+          "since": "2026-10-07T16:48:58.691388404Z"
+        },
+        {
+          "part": "bus",
+          "state": "healthy",
+          "since": "2026-10-07T16:48:58.691388404Z"
+        },
+        {
+          "part": "route",
+          "state": "healthy",
+          "since": "2026-10-07T16:48:58.691388404Z"
+        }
+      ]
+    }
+  },
+  {
+    "contract": 2,
+    "at": "2026-10-07T16:49:58.691388404Z",
+    "resources": [],
+    "network": {
+      "state": "healthy",
+      "since": "2026-10-07T16:48:58.691388404Z",
+      "parts": [
+        {
+          "part": "resolv-conf",
+          "state": "healthy",
+          "since": "2026-10-07T16:48:58.691388404Z",
+          "streak": 1
+        },
+        {
+          "part": "names",
+          "state": "healthy",
+          "since": "2026-10-07T16:48:58.691388404Z",
+          "streak": 1
+        },
+        {
+          "part": "bus",
+          "state": "healthy",
+          "since": "2026-10-07T16:48:58.691388404Z"
+        },
+        {
+          "part": "route",
+          "state": "healthy",
+          "since": "2026-10-07T16:48:58.691388404Z"
+        }
+      ]
+    }
+  },
+  {
+    "contract": 2,
+    "at": "2026-10-07T16:50:28.691388404Z",
+    "resources": [],
+    "network": {
+      "state": "unhealthy",
+      "since": "2026-10-07T16:50:28.691388404Z",
+      "parts": [
+        {
+          "part": "resolv-conf",
+          "state": "unhealthy",
+          "reason": "the resolver file was rewritten by another program",
+          "said": "/etc/resolv.conf differs from what networkmanager declares: it lists 192.0.2.53 where 10.77.0.2, 10.77.0.1 is declared; changed 2026-10-07T16:48:28Z; its own header names FortiClient",
+          "writer": "FortiClient",
+          "owner": "networkmanager",
+          "since": "2026-10-07T16:50:28.691388404Z",
+          "streak": 2
+        },
+        {
+          "part": "names",
+          "state": "unhealthy",
+          "reason": "neither mesh names nor public names resolve",
+          "said": "within 1s: 192.0.2.53 — anchor.internal (IPv4): no answer within 1s; anchor.internal (IPv6): no answer within 1s; example.com (IPv4): no answer within 1s",
+          "toward": [
+            "192.0.2.53"
+          ],
+          "since": "2026-10-07T16:50:28.691388404Z",
+          "streak": 2
+        },
+        {
+          "part": "bus",
+          "state": "healthy",
+          "since": "2026-10-07T16:48:58.691388404Z"
+        },
+        {
+          "part": "route",
+          "state": "healthy",
+          "since": "2026-10-07T16:48:58.691388404Z"
+        }
+      ]
+    }
+  },
+  {
+    "contract": 2,
+    "at": "2026-10-07T16:50:58.691388404Z",
+    "resources": [],
+    "network": {
+      "state": "healthy",
+      "since": "2026-10-07T16:50:58.691388404Z",
+      "parts": [
+        {
+          "part": "resolv-conf",
+          "state": "healthy",
+          "since": "2026-10-07T16:50:58.691388404Z"
+        },
+        {
+          "part": "names",
+          "state": "healthy",
+          "since": "2026-10-07T16:50:58.691388404Z"
+        },
+        {
+          "part": "bus",
+          "state": "healthy",
+          "since": "2026-10-07T16:48:58.691388404Z"
+        },
+        {
+          "part": "route",
+          "state": "healthy",
+          "since": "2026-10-07T16:48:58.691388404Z"
+        }
+      ]
+    }
+  }
+]

internal/inventory/health.go

@@ -40,6 +40,29 @@ type NodeHealth struct {
 	Resources []ResourceHealth
 	// Streaks is, per module, how many statements in a row said a resource of it was unhealthy.
 	Streaks map[string]int
+	// Network is the machine's own networking as its engine said it (novox/hq ADR 0241); nil from an
+	// engine older than that judging.
+	Network *NetworkHealth
+}
+
+// NetworkHealth is a machine's networking as its engine said it (ADR 0241).
+type NetworkHealth struct {
+	State string        `json:"state"`
+	Since time.Time     `json:"since"`
+	Parts []NetworkPart `json:"parts"`
+}
+
+// NetworkPart is one part of it: resolv-conf, names, tunnel, bus or route.
+type NetworkPart struct {
+	Part   string    `json:"part"`
+	State  string    `json:"state"`
+	Reason string    `json:"reason,omitempty"`
+	Said   string    `json:"said,omitempty"`
+	Writer string    `json:"writer,omitempty"`
+	Owner  string    `json:"owner,omitempty"`
+	Toward []string  `json:"toward,omitempty"`
+	Since  time.Time `json:"since"`
+	Streak int       `json:"streak,omitempty"`
 }
 
 // HealthOf is a machine's newest statement; false when its node-engine has never stated one.
@@ -60,7 +83,7 @@ func (i *Inventory) Healths(ctx context.Context) (map[string]NodeHealth, error)
 
 func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHealth, error) {
 	rows, err := i.store.Pool().Query(ctx,
-		`select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks
+		`select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks, h.network
 		 from node_health h join node n on n.id = h.node
 		 where $1 = '' or n.name = $1`, only)
 	if err != nil {
@@ -70,8 +93,8 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe
 	out := map[string]NodeHealth{}
 	for rows.Next() {
 		var h NodeHealth
-		var resources, streaks []byte
-		if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks); err != nil {
+		var resources, streaks, network []byte
+		if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks, &network); err != nil {
 			return nil, err
 		}
 		if err := json.Unmarshal(resources, &h.Resources); err != nil {
@@ -80,6 +103,11 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe
 		if err := json.Unmarshal(streaks, &h.Streaks); err != nil {
 			return nil, fmt.Errorf("%s's health cannot be read: %w", h.Node, err)
 		}
+		if len(network) > 0 {
+			if err := json.Unmarshal(network, &h.Network); err != nil {
+				return nil, fmt.Errorf("%s's network health cannot be read: %w", h.Node, err)
+			}
+		}
 		out[h.Node] = h
 	}
 	return out, rows.Err()
@@ -102,18 +130,25 @@ func (i *Inventory) RecordHealth(ctx context.Context, h NodeHealth) (bool, error
 	if err != nil {
 		return false, err
 	}
+	var network []byte
+	if h.Network != nil {
+		if network, err = json.Marshal(h.Network); err != nil {
+			return false, err
+		}
+	}
 	heard := h.HeardAt
 	if heard.IsZero() {
 		heard = time.Now()
 	}
 	var node string
 	err = i.store.Pool().QueryRow(ctx,
-		`insert into node_health (node, contract, said_at, heard_at, resources, streaks)
-		 select id, $2, $3, $4, $5, $6 from node where name = $1
+		`insert into node_health (node, contract, said_at, heard_at, resources, streaks, network)
+		 select id, $2, $3, $4, $5, $6, $7 from node where name = $1
 		 on conflict (node) do update set contract = excluded.contract, said_at = excluded.said_at,
-		     heard_at = excluded.heard_at, resources = excluded.resources, streaks = excluded.streaks
+		     heard_at = excluded.heard_at, resources = excluded.resources, streaks = excluded.streaks,
+		     network = excluded.network
 		   where node_health.said_at <= excluded.said_at
-		 returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks).Scan(&node)
+		 returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks, network).Scan(&node)
 	if errors.Is(err, pgx.ErrNoRows) {
 		if _, nerr := i.NodeByName(ctx, h.Node); nerr != nil {
 			return false, nerr

internal/inventory/migrations/0077-a-machine-says-how-its-network-is.sql

@@ -0,0 +1,10 @@
+-- A machine says how its network is (novox/hq ADR 0241, which extends ADR 0240 from what a module runs to
+-- the machine it runs on).
+--
+-- Beside the state of every long-running resource, each machine's node-engine states its own networking:
+-- the resolver file the uplink holder declared and who rewrote it, the names through every resolver it
+-- lists, the tunnel's handshake with the hub, the bus and the default route — the worst of them, since
+-- when, and each part. Kept with the machine's newest statement, replaced with it, so `node show`, the
+-- gate and a controller started again read the same word. Null for a machine whose node-engine is older
+-- than this judging: its network is not known — never healthy, never a reason to raise anything.
+alter table node_health add column network jsonb;

internal/link/protocol.go

@@ -428,6 +428,45 @@ type Health struct {
 	// At is when the engine looked, on the machine's clock: the order of its statements.
 	At        time.Time        `json:"at"`
 	Resources []ResourceHealth `json:"resources"`
+	// Network is the machine's own networking, judged by its engine (novox/hq ADR 0241); nil from an engine
+	// older than that judging, which is "not known", never healthy.
+	Network *NetworkHealth `json:"network,omitempty"`
+}
+
+// NetworkHealth is a machine's networking in one statement (ADR 0241): the worst of its parts, since
+// when, and each part. The node-engine's own (mesh-host internal/link NetworkHealth).
+type NetworkHealth struct {
+	State string        `json:"state"`
+	Since time.Time     `json:"since"`
+	Parts []NetworkPart `json:"parts"`
+}
+
+// The parts of a machine's networking its engine judges (ADR 0241).
+const (
+	PartResolvConf = "resolv-conf"
+	PartNames      = "names"
+	PartTunnel     = "tunnel"
+	PartBus        = "bus"
+	PartRoute      = "route"
+	// TowardHub is what a part failing toward the hub names in Toward.
+	TowardHub = "hub"
+)
+
+// NetworkPart is one part, as the engine judged it on its second look.
+type NetworkPart struct {
+	Part  string `json:"part"`
+	State string `json:"state"`
+	// Reason is in words with no address, path or domain; Said is the detail, kept as evidence.
+	Reason string `json:"reason,omitempty"`
+	Said   string `json:"said,omitempty"`
+	// Writer is the program that rewrote the resolver file, when the engine could name it; Owner the
+	// module whose file it is.
+	Writer string `json:"writer,omitempty"`
+	Owner  string `json:"owner,omitempty"`
+	// Toward is what a failure points at: "hub", or each resolver's address that failed.
+	Toward []string  `json:"toward,omitempty"`
+	Since  time.Time `json:"since"`
+	Streak int       `json:"streak,omitempty"`
 }
 
 // The states a resource is said in (ADR 0240 §4).