From 8bcf787258859703c4ed099d13896717e71d7b84 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 7 Oct 2026 18:52:16 +0200 Subject: [PATCH 1/2] Raise a machine's network from what its engine says, once (hq ADR 0241) A VPN client rewrote the laptop's resolver file and nothing said so. The engine now states its machine's networking; the controller keeps it with the machine's health (migration 0077) and raises the rewrite as its own finding naming the writer, the machine's own faults as machine..network, and what several machines cannot reach once, there. The gate waits on a rewrite it did not make rather than putting back a good build. --- cmd/mesh-controller/gate.go | 11 + cmd/mesh-controller/machine_network.go | 359 ++++++++++++++++++ cmd/mesh-controller/machine_network_test.go | 306 +++++++++++++++ cmd/mesh-controller/module_health.go | 21 +- cmd/mesh-controller/nodes.go | 3 + .../testdata/network-drill.json | 170 +++++++++ internal/inventory/health.go | 49 ++- ...0077-a-machine-says-how-its-network-is.sql | 10 + internal/link/protocol.go | 39 ++ 9 files changed, 959 insertions(+), 9 deletions(-) create mode 100644 cmd/mesh-controller/machine_network.go create mode 100644 cmd/mesh-controller/machine_network_test.go create mode 100644 cmd/mesh-controller/testdata/network-drill.json create mode 100644 internal/inventory/migrations/0077-a-machine-says-how-its-network-is.sql diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index 1c91964..750f81f 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -283,6 +283,8 @@ type machineWord struct { facts gateFacts on map[string]string whole string + // waiting is what holds the machine on something shown to be another's (ADR 0241): the judging waits. + waiting string } // kindCoreBehind is D10's kind: a machine runs core components older than the mesh holds, or has not @@ -346,6 +348,13 @@ func aboutTheMachine(machine string, moved []string, since time.Time, f gateFact } case coreBehind: // Not what the send moved: said nowhere against it. + case c.Kind == kindNetworkRewritten || c.Kind == kindNetworkUnreachable && c.Subject.ID != machine: + // **Shown to be somebody else's** (ADR 0241): another program rewrote the resolver file the send + // did not move, or the machine cannot reach another that is down. Nothing the send did; the + // judging waits for it rather than putting back a build at the bound. + if w.waiting == "" { + w.waiting = machine + "'s network: " + said + } default: if w.whole == "" { w.whole = machine + " as a whole: " + said @@ -433,6 +442,8 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs h, said = healthNotYet, on } else if w.whole != "" { h, said = healthNotYet, w.whole + } else if w.waiting != "" { + h, said = healthWaiting, w.waiting } } if h != healthGood && !slices.Contains(failing, j.module) { diff --git a/cmd/mesh-controller/machine_network.go b/cmd/mesh-controller/machine_network.go new file mode 100644 index 0000000..8fe0ce1 --- /dev/null +++ b/cmd/mesh-controller/machine_network.go @@ -0,0 +1,359 @@ +package main + +import ( + "context" + "fmt" + "slices" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A machine says how its network is (novox/hq ADR 0241, which extends ADR 0240 from what a module runs +// to the machine it runs on). +// +// **Every machine's node-engine judges its own networking** — the resolver file the uplink holder +// declared, the names through every resolver it lists, the tunnel's handshake with the hub, the bus, the +// default route — on the two-look rule, and states it beside its resources. The controller keeps the +// newest statement per machine and raises from all of them together: +// +// - **an outside writer of the resolver file is its own finding**, `machine...rewritten`: +// the file the uplink holder declares was rewritten by another program, named where the engine could +// name it. The names failing through what that program wrote are that finding's consequence, said in +// it — never a second condition; +// - **what is the machine's own** — its route, its tunnel, its resolvers answering wrong, a resolver +// that is no mesh machine — is `machine..network`; +// - **what points at another machine is said once, there** (the provider hold of ADR 0240 rule 5, for +// the network): a failure toward the hub or toward a mesh resolver is held under that machine when it +// is down on the record — silent, or its own network unhealthy — or when a second machine finds the +// same; then `machine..unreachable` names every machine that cannot reach it, and none of them +// raises anything of its own for it. One machine alone failing toward a healthy one is its own. +// +// Each is a warning; urgent on the control node, or when the bus cannot be reached, or for the hub. +// Cleared on the first statement that no longer says it. An engine older than this judging says nothing +// of its network, and nothing is raised for it. + +// The conditions a machine's network raises. +const ( + kindMachineNetwork = "machine-network" + kindNetworkRewritten = "network-rewritten" + kindNetworkUnreachable = "network-unreachable" + sourceNetwork = "network" +) + +// networkKinds are the kinds this judging owns: every open one it no longer says, it clears. +var networkKinds = []string{kindMachineNetwork, kindNetworkRewritten, kindNetworkUnreachable} + +// networkFacts is what one judging of every machine's network reads. +type networkFacts struct { + healths map[string]inventory.NodeHealth + // byAddress is each machine's address on the private network; hub the hub's name; control the + // control node's. + byAddress map[string]string + hub string + control string + // silent is every machine whose silence is an open condition. + silent map[string]bool +} + +// judgeNetworks raises and clears every machine's network conditions from every machine's newest +// statement, after one machine's statement was kept. +func judgeNetworks(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, now time.Time) error { + healths, err := inv.Healths(ctx) + if err != nil { + return err + } + overlays, err := inv.Overlays(ctx) + if err != nil { + return err + } + open, err := k.Open(ctx) + if err != nil { + return err + } + f := networkFacts{healths: healths, byAddress: map[string]string{}, control: controlHost(ctx, inv), + silent: map[string]bool{}} + for _, o := range overlays { + if o.Address != "" { + f.byAddress[o.Address] = o.Name + } + if o.Hub { + f.hub = o.Name + } + } + for _, c := range open { + if c.Subject.Scope == conditions.ScopeMachine && c.Kind == "silent" { + f.silent[c.Subject.ID] = true + } + } + var problems []string + said := map[string]bool{} + for _, o := range networkObservations(f) { + said[o.Key()] = true + if _, err := k.Observe(ctx, o); err != nil { + problems = append(problems, err.Error()) + } + } + for _, c := range open { + if !slices.Contains(networkKinds, c.Kind) || said[c.Key] { + continue + } + why := "no machine says it any more" + if c.Subject.Machine != "" { + why = c.Subject.Machine + "'s network no longer says it" + } + if _, err := k.Clear(ctx, c.Key, why); err != nil { + problems = append(problems, err.Error()) + } + } + if len(problems) > 0 { + return fmt.Errorf("%s", strings.Join(problems, "; ")) + } + return nil +} + +// pointed is one machine's failing part that points at another machine. +type pointed struct { + from string + part inventory.NetworkPart +} + +// networkObservations is every network condition the statements say now. Pure. +func networkObservations(f networkFacts) []conditions.Observation { + machines := make([]string, 0, len(f.healths)) + for m := range f.healths { + machines = append(machines, m) + } + sort.Strings(machines) + + unhealthy := func(m string) []inventory.NetworkPart { + h := f.healths[m] + if h.Network == nil { + return nil + } + var out []inventory.NetworkPart + for _, p := range h.Network.Parts { + if p.State == link.StateUnhealthy { + out = append(out, p) + } + } + return out + } + // The machines a part points at, other than its own: the hub, and each mesh resolver by its address. + // An address that is no mesh machine's — the resolver a VPN client wrote in — is the machine's own. + targets := func(m string, p inventory.NetworkPart) ([]string, bool) { + if len(p.Toward) == 0 { + return nil, false + } + var out []string + for _, t := range p.Toward { + x := f.byAddress[t] + if t == link.TowardHub { + x = f.hub + } + if x == "" || x == m { + return nil, false + } + if !slices.Contains(out, x) { + out = append(out, x) + } + } + return out, true + } + + // First pass: what points at whom. + pointing := map[string][]pointed{} + for _, m := range machines { + for _, p := range unhealthy(m) { + if xs, ok := targets(m, p); ok { + for _, x := range xs { + pointing[x] = append(pointing[x], pointed{from: m, part: p}) + } + } + } + } + from := func(x string) []string { + var out []string + for _, pt := range pointing[x] { + if !slices.Contains(out, pt.from) { + out = append(out, pt.from) + } + } + sort.Strings(out) + return out + } + // A machine is down on the record when its silence is open or its own network is unhealthy, or when + // two machines find it unreachable: then what points at it is held there. + down := func(x string) bool { + return f.silent[x] || len(unhealthy(x)) > 0 || len(from(x)) >= 2 + } + + var out []conditions.Observation + saysOwn := map[string]bool{} + for _, m := range machines { + parts := unhealthy(m) + if len(parts) == 0 { + continue + } + var own []inventory.NetworkPart + var rewritten *inventory.NetworkPart + for i, p := range parts { + if p.Part == link.PartResolvConf { + rewritten = &parts[i] + continue + } + if xs, ok := targets(m, p); ok && allDown(xs, down) { + continue // held at the machines it points at + } + own = append(own, p) + } + if rewritten != nil { + out = append(out, rewrittenObservation(m, *rewritten, parts, f)) + // The names failing through what another program wrote are that finding's, said in it. + kept := own[:0] + for _, p := range own { + if p.Part != link.PartNames { + kept = append(kept, p) + } + } + own = kept + } + if len(own) > 0 { + out = append(out, machineNetworkObservation(m, own, f, from(m))) + saysOwn[m] = true + } + } + // Said once, at the machine everybody points at — unless its own network condition already says it + // (listed there), or its silence does. + targetsSorted := make([]string, 0, len(pointing)) + for x := range pointing { + targetsSorted = append(targetsSorted, x) + } + sort.Strings(targetsSorted) + for _, x := range targetsSorted { + if !down(x) || saysOwn[x] || f.silent[x] { + continue + } + out = append(out, unreachableObservation(x, pointing[x], from(x), f)) + } + return out +} + +func allDown(xs []string, down func(string) bool) bool { + for _, x := range xs { + if !down(x) { + return false + } + } + return len(xs) > 0 +} + +// rewrittenObservation is the resolver file rewritten by another program: its own finding, naming the +// writer where the engine could, and what it costs the machine. +func rewrittenObservation(m string, p inventory.NetworkPart, all []inventory.NetworkPart, f networkFacts) conditions.Observation { + writer := "" + if p.Writer != "" { + writer = " (" + p.Writer + ")" + } + cost := "the names through it are not yet judged" + said := []string{p.Part + ": " + p.Said} + for _, q := range all { + if q.Part == link.PartNames { + cost = strings.TrimSuffix(q.Reason, ".") + said = append(said, q.Part+": "+q.Said) + } + } + if cost == "the names through it are not yet judged" { + cost = "the names still resolve through what it wrote" + } + id := m + if p.Owner != "" { + // Named by the module whose file it is: a send that moved that module is what the gate + // holds it on (issue 281's rule — what names a moved module is that module's). + id = m + "." + p.Owner + } + severity := conditions.Warning + if m == f.control { + severity = conditions.Urgent + } + summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the "+ + "node-engine writes it back at its next reconcile, or that program gives it back", m, writer, cost) + if p.Owner != "" { + summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until "+ + "the node-engine writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost) + } + return conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "rewritten", Kind: kindNetworkRewritten, + Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary, + Said: fmt.Sprintf("since %s: %s", p.Since.UTC().Format("2006-01-02 15:04:05 MST"), strings.Join(said, " | "))} +} + +// machineNetworkObservation is what is wrong with a machine's own networking. +func machineNetworkObservation(m string, parts []inventory.NetworkPart, f networkFacts, waiting []string) conditions.Observation { + var words, said []string + severity := conditions.Warning + for _, p := range parts { + words = append(words, p.Reason) + said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, p.Since.UTC().Format("2006-01-02 15:04:05 MST"), p.Said)) + if p.Part == link.PartBus { + severity = conditions.Urgent + } + } + if m == f.control || m == f.hub { + severity = conditions.Urgent + } + summary := fmt.Sprintf("%s's network is not healthy: %s", m, strings.Join(words, "; ")) + if len(waiting) > 0 { + severity = conditions.Urgent + summary += fmt.Sprintf("; %s cannot reach it", strings.Join(waiting, ", ")) + } + return conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Token: "network", Kind: kindMachineNetwork, + Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary, Said: strings.Join(said, " | ")} +} + +// unreachableObservation is one machine others cannot reach, said once there. +func unreachableObservation(x string, pts []pointed, from []string, f networkFacts) conditions.Observation { + var what []string + var said []string + for _, pt := range pts { + w := map[string]string{link.PartTunnel: "the tunnel to it", link.PartBus: "the bus on it", + link.PartNames: "its resolver"}[pt.part.Part] + if w == "" { + w = pt.part.Part + } + if !slices.Contains(what, w) { + what = append(what, w) + } + said = append(said, fmt.Sprintf("%s: %s: %s", pt.from, pt.part.Part, pt.part.Said)) + } + severity := conditions.Warning + if x == f.hub || x == f.control || slices.Contains(what, "the bus on it") { + severity = conditions.Urgent + } + return conditions.Observation{Scope: conditions.ScopeMachine, ID: x, Token: "unreachable", Kind: kindNetworkUnreachable, + Machine: x, Also: from, Severity: severity, Source: sourceNetwork, + Summary: fmt.Sprintf("%s cannot be reached from %s: %s", x, strings.Join(from, ", "), strings.Join(what, ", ")), + Said: strings.Join(said, " | ")} +} + +// networkLines is what `node show` says of a machine's networking. +func networkLines(h inventory.NodeHealth, had bool, now time.Time) []string { + if !had || h.Network == nil { + return []string{" its node-engine does not say how its network is — it is older than that judging (ADR 0241)"} + } + out := []string{fmt.Sprintf(" its network: %s since %s", h.Network.State, h.Network.Since.Local().Format("2006-01-02 15:04"))} + for _, p := range h.Network.Parts { + line := fmt.Sprintf(" %-10s %s", p.State, p.Part) + if p.Reason != "" { + line += " — " + p.Reason + } + if p.Writer != "" { + line += " (" + p.Writer + ")" + } + out = append(out, line) + } + return out +} diff --git a/cmd/mesh-controller/machine_network_test.go b/cmd/mesh-controller/machine_network_test.go new file mode 100644 index 0000000..20bea9d --- /dev/null +++ b/cmd/mesh-controller/machine_network_test.go @@ -0,0 +1,306 @@ +package main + +import ( + "encoding/json" + "os" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A machine says how its network is (novox/hq ADR 0241, "how it is checked"): the resolver file rewritten +// by another program is one finding, naming the writer, with the names it costs said in it; what is the +// machine's own is `machine..network`; what points at another machine that is down is said once, there; +// one machine alone failing toward a healthy one is its own; the control node, the hub and the bus are +// urgent; an engine that says nothing of its network raises nothing; and the gate waits on what is shown to +// be another's. + +func aNetwork(state string, parts ...inventory.NetworkPart) *inventory.NetworkHealth { + for i := range parts { + if parts[i].State == "" { + parts[i].State = link.StateUnhealthy + } + parts[i].Since = h0 + } + return &inventory.NetworkHealth{State: state, Since: h0, Parts: parts} +} + +func netFacts(healths map[string]*inventory.NetworkHealth) networkFacts { + f := networkFacts{healths: map[string]inventory.NodeHealth{}, hub: "anchor", control: "anchor", + byAddress: map[string]string{"10.77.0.1": "anchor", "10.77.0.2": "laptop", "10.77.0.3": "spare"}, + silent: map[string]bool{}} + for m, n := range healths { + f.healths[m] = inventory.NodeHealth{Node: m, Network: n} + } + return f +} + +var ( + rewrittenByVPN = inventory.NetworkPart{Part: link.PartResolvConf, Reason: "the resolver file was rewritten by another program", + Said: "/etc/resolv.conf lists 172.16.5.5 where 10.77.0.1 is declared", Writer: "FortiClient", Owner: "networkmanager"} + namesThroughVPN = inventory.NetworkPart{Part: link.PartNames, Reason: "mesh names do not resolve", + Said: "172.16.5.5 — anchor.internal (IPv4): says no such name", Toward: []string{"172.16.5.5"}} + tunnelDown = inventory.NetworkPart{Part: link.PartTunnel, Reason: "the tunnel to the hub has not handshaken for over five minutes", + Said: "mesh0's newest handshake with the hub was 9m0s ago", Toward: []string{link.TowardHub}} + noRoute = inventory.NetworkPart{Part: link.PartRoute, Reason: "the machine has no default route", Said: "no default route"} +) + +func keysOf(obs []conditions.Observation) []string { + var keys []string + for _, o := range obs { + keys = append(keys, o.Key()) + } + return keys +} + +func TestAResolverFileRewrittenIsOneFindingNamingItsWriterAndWhatItCosts(t *testing.T) { + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, rewrittenByVPN, namesThroughVPN), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" { + t.Fatalf("want one finding, the rewrite, got %v", keys) + } + o := obs[0] + for _, want := range []string{"laptop", "rewritten by another program (FortiClient)", "mesh names do not resolve", + "next reconcile"} { + if !strings.Contains(o.Summary, want) { + t.Errorf("the summary does not say %q: %s", want, o.Summary) + } + } + if strings.Contains(o.Summary, "172.16.") || strings.Contains(o.Summary, "/etc/") { + t.Errorf("an address or a path reached the summary: %s", o.Summary) + } + if !strings.Contains(o.Said, "172.16.5.5") || o.Severity != conditions.Warning || o.Machine != "laptop" { + t.Errorf("the evidence or the severity is wrong: %+v", o) + } +} + +func TestOneMachineFailingTowardAHealthyHubIsItsOwn(t *testing.T) { + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, tunnelDown), + "spare": aNetwork(link.StateHealthy), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" { + t.Fatalf("want the laptop's own, got %v", keys) + } + if obs[0].Severity != conditions.Warning { + t.Fatalf("a laptop's own tunnel is a warning, got %s", obs[0].Severity) + } +} + +func TestTwoMachinesThatCannotReachTheHubAreSaidOnceAtTheHub(t *testing.T) { + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, tunnelDown), + "spare": aNetwork(link.StateUnhealthy, tunnelDown), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.unreachable" { + t.Fatalf("want one condition at the hub, got %v", keys) + } + o := obs[0] + if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop, spare") || len(o.Also) != 2 { + t.Fatalf("the hub's condition is %+v", o) + } +} + +func TestWhatPointsAtASilentHubIsHeldUnderItsSilence(t *testing.T) { + f := netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, tunnelDown)}) + f.silent["anchor"] = true + if obs := networkObservations(f); len(obs) != 0 { + t.Fatalf("the hub's silence says it; got %v", keysOf(obs)) + } +} + +func TestAHubWhoseOwnNetworkIsUnhealthyListsWhoCannotReachIt(t *testing.T) { + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateUnhealthy, noRoute), + "laptop": aNetwork(link.StateUnhealthy, tunnelDown), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.network" { + t.Fatalf("want the hub's own, holding the laptop's, got %v", keys) + } + if o := obs[0]; o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop cannot reach it") { + t.Fatalf("the hub's condition is %+v", o) + } +} + +func TestOneMachineFailingAHealthyMeshResolverIsItsOwnAndTwoAreTheResolvers(t *testing.T) { + silentResolver := inventory.NetworkPart{Part: link.PartNames, Reason: "1 of its 2 resolvers do not answer as the mesh's do", + Said: "10.77.0.3 — no answer within 1s", Toward: []string{"10.77.0.3"}} + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateHealthy), "spare": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, silentResolver), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" { + t.Fatalf("one machine alone: want its own, got %v", keys) + } + obs = networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateUnhealthy, silentResolver), "spare": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, silentResolver), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.spare.unreachable" { + t.Fatalf("two machines: want it said once at the resolver's machine, got %v", keys) + } + if !strings.Contains(obs[0].Summary, "its resolver") { + t.Fatalf("the resolver's machine is said %s", obs[0].Summary) + } +} + +func TestTheControlNodeAndTheBusAreUrgent(t *testing.T) { + f := netFacts(map[string]*inventory.NetworkHealth{"anchor": aNetwork(link.StateUnhealthy, rewrittenByVPN)}) + if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent { + t.Fatalf("the control node's rewritten file: %+v", obs) + } + bus := inventory.NetworkPart{Part: link.PartBus, Reason: "the bus cannot be reached", Said: "no link"} + f = netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, bus, noRoute)}) + if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent { + t.Fatalf("the bus unreachable from the laptop: %+v", obs) + } +} + +func TestAnEngineThatSaysNothingOfItsNetworkRaisesNothing(t *testing.T) { + f := netFacts(map[string]*inventory.NetworkHealth{"laptop": nil, "anchor": aNetwork(link.StateHealthy)}) + if obs := networkObservations(f); len(obs) != 0 { + t.Fatalf("got %v", keysOf(obs)) + } +} + +// Through the store and the keeper: the statement kept, the rewrite raised from it, cleared when the file +// is written back, and node show saying it. +func TestARewrittenResolverFileIsRaisedFromTheStatementAndClearedWhenWrittenBack(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv, k := open.inventory, conditionsFrom + say := func(at time.Time, n *link.NetworkHealth) { + t.Helper() + if err := stateHealth(ctx, inv, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { + t.Fatal(err) + } + } + openKeys := func() []string { + t.Helper() + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + var keys []string + for _, c := range list { + keys = append(keys, c.Key) + } + return keys + } + healthy := &link.NetworkHealth{State: link.StateHealthy, Since: h0, Parts: []link.NetworkPart{ + {Part: link.PartResolvConf, State: link.StateHealthy, Since: h0}}} + rewritten := &link.NetworkHealth{State: link.StateUnhealthy, Since: h0.Add(time.Minute), Parts: []link.NetworkPart{ + {Part: link.PartResolvConf, State: link.StateUnhealthy, Reason: rewrittenByVPN.Reason, Said: rewrittenByVPN.Said, + Writer: "FortiClient", Owner: "networkmanager", Since: h0.Add(time.Minute)}, + {Part: link.PartNames, State: link.StateUnhealthy, Reason: namesThroughVPN.Reason, Said: namesThroughVPN.Said, + Toward: namesThroughVPN.Toward, Since: h0.Add(time.Minute)}}} + + say(h0, healthy) + if keys := openKeys(); len(keys) != 0 { + t.Fatalf("a healthy network raised %v", keys) + } + say(h0.Add(time.Minute), rewritten) + if keys := openKeys(); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" { + t.Fatalf("the rewrite raised %v", keys) + } + kept, had, err := inv.HealthOf(ctx, "laptop") + if err != nil || !had || kept.Network == nil || kept.Network.Parts[0].Writer != "FortiClient" { + t.Fatalf("the statement's network was not kept: %+v %v", kept.Network, err) + } + if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "FortiClient") || + !strings.Contains(lines, "unhealthy resolv-conf") { + t.Fatalf("node show says:\n%s", lines) + } + say(h0.Add(2*time.Minute), healthy) + if keys := openKeys(); len(keys) != 0 { + t.Fatalf("written back, still open: %v", keys) + } + // An engine older than the judging says no network: nothing raised, and node show says it is not known. + say(h0.Add(3*time.Minute), nil) + kept, had, _ = inv.HealthOf(ctx, "laptop") + if keys := openKeys(); len(keys) != 0 || kept.Network != nil { + t.Fatalf("an older engine: %v %+v", keys, kept.Network) + } + if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "older than that judging") { + t.Fatalf("node show says:\n%s", lines) + } +} + +// The gate: a resolver file another program rewrote waits the judging rather than failing it at the +// bound; the same, when the send moved the module whose file it is, is that module's; a machine's own +// network fault holds the machine as a whole, as before. +func TestTheGateWaitsOnARewriteItDidNotMakeAndHoldsTheOwnerOnOneItMoved(t *testing.T) { + since := h0 + rewrite := conditions.Condition{Key: "machine.laptop.networkmanager.rewritten", Kind: kindNetworkRewritten, + Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop.networkmanager", Machine: "laptop"}, + Summary: "the resolver file was rewritten", Raised: since.Add(time.Minute), Source: sourceNetwork} + f := gateFacts{judged: true, open: []conditions.Condition{rewrite}} + if w := aboutTheMachine("laptop", []string{"letta"}, since, f); w.waiting == "" || w.whole != "" || len(w.on) != 0 { + t.Fatalf("a rewrite the send did not make: %+v", w) + } + if w := aboutTheMachine("laptop", []string{"networkmanager", "letta"}, since, f); w.on["networkmanager"] == "" || + w.on["letta"] != "" || w.waiting != "" { + t.Fatalf("a rewrite of the file a moved module owns: %+v", w) + } + own := conditions.Condition{Key: "machine.laptop.network", Kind: kindMachineNetwork, + Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"}, + Summary: "laptop's network is not healthy", Raised: since.Add(time.Minute), Source: sourceNetwork} + if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true, + open: []conditions.Condition{own}}); w.whole == "" || w.waiting != "" { + t.Fatalf("the machine's own network: %+v", w) + } + unreachable := conditions.Condition{Key: "machine.anchor.unreachable", Kind: kindNetworkUnreachable, + Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor", Also: []string{"laptop", "spare"}}, + Summary: "anchor cannot be reached", Raised: since.Add(time.Minute), Source: sourceNetwork} + if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true, + open: []conditions.Condition{unreachable}}); w.waiting == "" || w.whole != "" { + t.Fatalf("a machine that cannot reach the hub: %+v", w) + } +} + +// TestTheDrillsStatementsRaiseAndClearTheRewrite replays the drill of ADR 0241 (mesh-host +// internal/network TestDrill…): what a node-engine said in a throwaway container while its resolver file +// was declared, rewritten as a VPN client rewrites it, and written back — recorded, with the mesh's names +// and addresses replaced by this test mesh's. Healthy raises nothing; the rewrite, on its second look, is +// raised as one finding naming the writer; written back, it clears. +func TestTheDrillsStatementsRaiseAndClearTheRewrite(t *testing.T) { + raw, err := os.ReadFile("testdata/network-drill.json") + if err != nil { + t.Fatal(err) + } + var said []link.Health + if err := json.Unmarshal(raw, &said); err != nil { + t.Fatal(err) + } + open := aMesh(t) + ctx := t.Context() + k := conditionsFrom + var raisedAt []int + for i, h := range said { + if err := stateHealth(ctx, open.inventory, k, "laptop", h, h.At); err != nil { + t.Fatal(err) + } + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + for _, c := range list { + if c.Key != "machine.laptop.networkmanager.rewritten" || !strings.Contains(c.Summary, "(FortiClient)") { + t.Fatalf("statement %d raised %s: %s", i, c.Key, c.Summary) + } + raisedAt = append(raisedAt, i) + } + } + // Five statements: healthy, healthy, one failing look (still healthy), unhealthy, written back. + if len(raisedAt) != 1 || raisedAt[0] != 3 { + t.Fatalf("the rewrite was open after statements %v; want after the fourth alone", raisedAt) + } +} diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index 23ace29..9aa8217 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -83,8 +83,16 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke for module := range unhealthy { streaks[module] = prev.Streaks[module] + 1 } + var network *inventory.NetworkHealth + if h.Network != nil { + network = &inventory.NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: []inventory.NetworkPart{}} + for _, p := range h.Network.Parts { + network.Parts = append(network.Parts, inventory.NetworkPart{Part: p.Part, State: p.State, Reason: p.Reason, + Said: p.Said, Writer: p.Writer, Owner: p.Owner, Toward: p.Toward, Since: p.Since, Streak: p.Streak}) + } + } stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At, - HeardAt: now, Resources: resources, Streaks: streaks}) + HeardAt: now, Resources: resources, Streaks: streaks, Network: network}) if err != nil || !stored { if err == nil { healthRefused.Add(1) @@ -94,7 +102,16 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke if k == nil { return nil } - return judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now) + err = judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now) + // And every machine's network, from every machine's newest statement (ADR 0241): a statement about one + // machine can hold another's finding, or release it. + if nerr := judgeNetworks(ctx, inv, k, now); nerr != nil { + if err == nil { + return nerr + } + return fmt.Errorf("%w; %v", err, nerr) + } + return err } // judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a diff --git a/cmd/mesh-controller/nodes.go b/cmd/mesh-controller/nodes.go index 701b010..8c63a06 100644 --- a/cmd/mesh-controller/nodes.go +++ b/cmd/mesh-controller/nodes.go @@ -546,6 +546,9 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error for _, line := range healthLines(h, had, time.Now()) { fmt.Println(line) } + for _, line := range networkLines(h, had, time.Now()) { + fmt.Println(line) + } } held, err := inv.Profile(ctx, name) diff --git a/cmd/mesh-controller/testdata/network-drill.json b/cmd/mesh-controller/testdata/network-drill.json new file mode 100644 index 0000000..edf7321 --- /dev/null +++ b/cmd/mesh-controller/testdata/network-drill.json @@ -0,0 +1,170 @@ +[ + { + "contract": 2, + "at": "2026-10-07T16:48:58.691388404Z", + "resources": [], + "network": { + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z", + "parts": [ + { + "part": "resolv-conf", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + }, + { + "part": "names", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + }, + { + "part": "bus", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + }, + { + "part": "route", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + } + ] + } + }, + { + "contract": 2, + "at": "2026-10-07T16:49:28.691388404Z", + "resources": [], + "network": { + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z", + "parts": [ + { + "part": "resolv-conf", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + }, + { + "part": "names", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + }, + { + "part": "bus", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + }, + { + "part": "route", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + } + ] + } + }, + { + "contract": 2, + "at": "2026-10-07T16:49:58.691388404Z", + "resources": [], + "network": { + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z", + "parts": [ + { + "part": "resolv-conf", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z", + "streak": 1 + }, + { + "part": "names", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z", + "streak": 1 + }, + { + "part": "bus", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + }, + { + "part": "route", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + } + ] + } + }, + { + "contract": 2, + "at": "2026-10-07T16:50:28.691388404Z", + "resources": [], + "network": { + "state": "unhealthy", + "since": "2026-10-07T16:50:28.691388404Z", + "parts": [ + { + "part": "resolv-conf", + "state": "unhealthy", + "reason": "the resolver file was rewritten by another program", + "said": "/etc/resolv.conf differs from what networkmanager declares: it lists 192.0.2.53 where 10.77.0.2, 10.77.0.1 is declared; changed 2026-10-07T16:48:28Z; its own header names FortiClient", + "writer": "FortiClient", + "owner": "networkmanager", + "since": "2026-10-07T16:50:28.691388404Z", + "streak": 2 + }, + { + "part": "names", + "state": "unhealthy", + "reason": "neither mesh names nor public names resolve", + "said": "within 1s: 192.0.2.53 — anchor.internal (IPv4): no answer within 1s; anchor.internal (IPv6): no answer within 1s; example.com (IPv4): no answer within 1s", + "toward": [ + "192.0.2.53" + ], + "since": "2026-10-07T16:50:28.691388404Z", + "streak": 2 + }, + { + "part": "bus", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + }, + { + "part": "route", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + } + ] + } + }, + { + "contract": 2, + "at": "2026-10-07T16:50:58.691388404Z", + "resources": [], + "network": { + "state": "healthy", + "since": "2026-10-07T16:50:58.691388404Z", + "parts": [ + { + "part": "resolv-conf", + "state": "healthy", + "since": "2026-10-07T16:50:58.691388404Z" + }, + { + "part": "names", + "state": "healthy", + "since": "2026-10-07T16:50:58.691388404Z" + }, + { + "part": "bus", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + }, + { + "part": "route", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + } + ] + } + } +] \ No newline at end of file diff --git a/internal/inventory/health.go b/internal/inventory/health.go index a350080..fe31d5c 100644 --- a/internal/inventory/health.go +++ b/internal/inventory/health.go @@ -40,6 +40,29 @@ type NodeHealth struct { Resources []ResourceHealth // Streaks is, per module, how many statements in a row said a resource of it was unhealthy. Streaks map[string]int + // Network is the machine's own networking as its engine said it (novox/hq ADR 0241); nil from an + // engine older than that judging. + Network *NetworkHealth +} + +// NetworkHealth is a machine's networking as its engine said it (ADR 0241). +type NetworkHealth struct { + State string `json:"state"` + Since time.Time `json:"since"` + Parts []NetworkPart `json:"parts"` +} + +// NetworkPart is one part of it: resolv-conf, names, tunnel, bus or route. +type NetworkPart struct { + Part string `json:"part"` + State string `json:"state"` + Reason string `json:"reason,omitempty"` + Said string `json:"said,omitempty"` + Writer string `json:"writer,omitempty"` + Owner string `json:"owner,omitempty"` + Toward []string `json:"toward,omitempty"` + Since time.Time `json:"since"` + Streak int `json:"streak,omitempty"` } // HealthOf is a machine's newest statement; false when its node-engine has never stated one. @@ -60,7 +83,7 @@ func (i *Inventory) Healths(ctx context.Context) (map[string]NodeHealth, error) func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHealth, error) { rows, err := i.store.Pool().Query(ctx, - `select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks + `select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks, h.network from node_health h join node n on n.id = h.node where $1 = '' or n.name = $1`, only) if err != nil { @@ -70,8 +93,8 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe out := map[string]NodeHealth{} for rows.Next() { var h NodeHealth - var resources, streaks []byte - if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks); err != nil { + var resources, streaks, network []byte + if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks, &network); err != nil { return nil, err } if err := json.Unmarshal(resources, &h.Resources); err != nil { @@ -80,6 +103,11 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe if err := json.Unmarshal(streaks, &h.Streaks); err != nil { return nil, fmt.Errorf("%s's health cannot be read: %w", h.Node, err) } + if len(network) > 0 { + if err := json.Unmarshal(network, &h.Network); err != nil { + return nil, fmt.Errorf("%s's network health cannot be read: %w", h.Node, err) + } + } out[h.Node] = h } return out, rows.Err() @@ -102,18 +130,25 @@ func (i *Inventory) RecordHealth(ctx context.Context, h NodeHealth) (bool, error if err != nil { return false, err } + var network []byte + if h.Network != nil { + if network, err = json.Marshal(h.Network); err != nil { + return false, err + } + } heard := h.HeardAt if heard.IsZero() { heard = time.Now() } var node string err = i.store.Pool().QueryRow(ctx, - `insert into node_health (node, contract, said_at, heard_at, resources, streaks) - select id, $2, $3, $4, $5, $6 from node where name = $1 + `insert into node_health (node, contract, said_at, heard_at, resources, streaks, network) + select id, $2, $3, $4, $5, $6, $7 from node where name = $1 on conflict (node) do update set contract = excluded.contract, said_at = excluded.said_at, - heard_at = excluded.heard_at, resources = excluded.resources, streaks = excluded.streaks + heard_at = excluded.heard_at, resources = excluded.resources, streaks = excluded.streaks, + network = excluded.network where node_health.said_at <= excluded.said_at - returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks).Scan(&node) + returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks, network).Scan(&node) if errors.Is(err, pgx.ErrNoRows) { if _, nerr := i.NodeByName(ctx, h.Node); nerr != nil { return false, nerr diff --git a/internal/inventory/migrations/0077-a-machine-says-how-its-network-is.sql b/internal/inventory/migrations/0077-a-machine-says-how-its-network-is.sql new file mode 100644 index 0000000..2da375f --- /dev/null +++ b/internal/inventory/migrations/0077-a-machine-says-how-its-network-is.sql @@ -0,0 +1,10 @@ +-- A machine says how its network is (novox/hq ADR 0241, which extends ADR 0240 from what a module runs to +-- the machine it runs on). +-- +-- Beside the state of every long-running resource, each machine's node-engine states its own networking: +-- the resolver file the uplink holder declared and who rewrote it, the names through every resolver it +-- lists, the tunnel's handshake with the hub, the bus and the default route — the worst of them, since +-- when, and each part. Kept with the machine's newest statement, replaced with it, so `node show`, the +-- gate and a controller started again read the same word. Null for a machine whose node-engine is older +-- than this judging: its network is not known — never healthy, never a reason to raise anything. +alter table node_health add column network jsonb; diff --git a/internal/link/protocol.go b/internal/link/protocol.go index e7767f7..ad27bec 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -428,6 +428,45 @@ type Health struct { // At is when the engine looked, on the machine's clock: the order of its statements. At time.Time `json:"at"` Resources []ResourceHealth `json:"resources"` + // Network is the machine's own networking, judged by its engine (novox/hq ADR 0241); nil from an engine + // older than that judging, which is "not known", never healthy. + Network *NetworkHealth `json:"network,omitempty"` +} + +// NetworkHealth is a machine's networking in one statement (ADR 0241): the worst of its parts, since +// when, and each part. The node-engine's own (mesh-host internal/link NetworkHealth). +type NetworkHealth struct { + State string `json:"state"` + Since time.Time `json:"since"` + Parts []NetworkPart `json:"parts"` +} + +// The parts of a machine's networking its engine judges (ADR 0241). +const ( + PartResolvConf = "resolv-conf" + PartNames = "names" + PartTunnel = "tunnel" + PartBus = "bus" + PartRoute = "route" + // TowardHub is what a part failing toward the hub names in Toward. + TowardHub = "hub" +) + +// NetworkPart is one part, as the engine judged it on its second look. +type NetworkPart struct { + Part string `json:"part"` + State string `json:"state"` + // Reason is in words with no address, path or domain; Said is the detail, kept as evidence. + Reason string `json:"reason,omitempty"` + Said string `json:"said,omitempty"` + // Writer is the program that rewrote the resolver file, when the engine could name it; Owner the + // module whose file it is. + Writer string `json:"writer,omitempty"` + Owner string `json:"owner,omitempty"` + // Toward is what a failure points at: "hub", or each resolver's address that failed. + Toward []string `json:"toward,omitempty"` + Since time.Time `json:"since"` + Streak int `json:"streak,omitempty"` } // The states a resource is said in (ADR 0240 §4). From b8bbf9c79fd82033c228717ec56cf00d5e127baf Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 7 Oct 2026 18:53:21 +0200 Subject: [PATCH 2/2] Hold the network statement's field names on the controller's side (hq ADR 0241) --- internal/link/protocol_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/link/protocol_test.go b/internal/link/protocol_test.go index 4f8a412..cc2120a 100644 --- a/internal/link/protocol_test.go +++ b/internal/link/protocol_test.go @@ -33,6 +33,13 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { []string{"module", "resource", "kind", "target", "state", "reason", "since", "streak", "restarts", "check", "needs"}}, {HealthSaid{Node: "n"}, []string{"node", "health"}}, + // novox/hq ADR 0241: the machine's own networking, beside its resources. + {Health{Contract: ReadinessContract, Resources: []ResourceHealth{}, Network: &NetworkHealth{State: "unhealthy", + Parts: []NetworkPart{}}}, []string{"contract", "at", "resources", "network"}}, + {NetworkHealth{State: "unhealthy", Parts: []NetworkPart{}}, []string{"state", "since", "parts"}}, + {NetworkPart{Part: "resolv-conf", State: "unhealthy", Reason: "r", Said: "s", Writer: "w", Owner: "o", + Toward: []string{"hub"}, Streak: 2}, []string{"part", "state", "reason", "said", "writer", "owner", "toward", + "since", "streak"}}, } { raw, err := json.Marshal(c.value) if err != nil {