diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index 1c91964..750f81f 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -283,6 +283,8 @@ type machineWord struct { facts gateFacts on map[string]string whole string + // waiting is what holds the machine on something shown to be another's (ADR 0241): the judging waits. + waiting string } // kindCoreBehind is D10's kind: a machine runs core components older than the mesh holds, or has not @@ -346,6 +348,13 @@ func aboutTheMachine(machine string, moved []string, since time.Time, f gateFact } case coreBehind: // Not what the send moved: said nowhere against it. + case c.Kind == kindNetworkRewritten || c.Kind == kindNetworkUnreachable && c.Subject.ID != machine: + // **Shown to be somebody else's** (ADR 0241): another program rewrote the resolver file the send + // did not move, or the machine cannot reach another that is down. Nothing the send did; the + // judging waits for it rather than putting back a build at the bound. + if w.waiting == "" { + w.waiting = machine + "'s network: " + said + } default: if w.whole == "" { w.whole = machine + " as a whole: " + said @@ -433,6 +442,8 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs h, said = healthNotYet, on } else if w.whole != "" { h, said = healthNotYet, w.whole + } else if w.waiting != "" { + h, said = healthWaiting, w.waiting } } if h != healthGood && !slices.Contains(failing, j.module) { diff --git a/cmd/mesh-controller/machine_network.go b/cmd/mesh-controller/machine_network.go new file mode 100644 index 0000000..8fe0ce1 --- /dev/null +++ b/cmd/mesh-controller/machine_network.go @@ -0,0 +1,359 @@ +package main + +import ( + "context" + "fmt" + "slices" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A machine says how its network is (novox/hq ADR 0241, which extends ADR 0240 from what a module runs +// to the machine it runs on). +// +// **Every machine's node-engine judges its own networking** — the resolver file the uplink holder +// declared, the names through every resolver it lists, the tunnel's handshake with the hub, the bus, the +// default route — on the two-look rule, and states it beside its resources. The controller keeps the +// newest statement per machine and raises from all of them together: +// +// - **an outside writer of the resolver file is its own finding**, `machine...rewritten`: +// the file the uplink holder declares was rewritten by another program, named where the engine could +// name it. The names failing through what that program wrote are that finding's consequence, said in +// it — never a second condition; +// - **what is the machine's own** — its route, its tunnel, its resolvers answering wrong, a resolver +// that is no mesh machine — is `machine..network`; +// - **what points at another machine is said once, there** (the provider hold of ADR 0240 rule 5, for +// the network): a failure toward the hub or toward a mesh resolver is held under that machine when it +// is down on the record — silent, or its own network unhealthy — or when a second machine finds the +// same; then `machine..unreachable` names every machine that cannot reach it, and none of them +// raises anything of its own for it. One machine alone failing toward a healthy one is its own. +// +// Each is a warning; urgent on the control node, or when the bus cannot be reached, or for the hub. +// Cleared on the first statement that no longer says it. An engine older than this judging says nothing +// of its network, and nothing is raised for it. + +// The conditions a machine's network raises. +const ( + kindMachineNetwork = "machine-network" + kindNetworkRewritten = "network-rewritten" + kindNetworkUnreachable = "network-unreachable" + sourceNetwork = "network" +) + +// networkKinds are the kinds this judging owns: every open one it no longer says, it clears. +var networkKinds = []string{kindMachineNetwork, kindNetworkRewritten, kindNetworkUnreachable} + +// networkFacts is what one judging of every machine's network reads. +type networkFacts struct { + healths map[string]inventory.NodeHealth + // byAddress is each machine's address on the private network; hub the hub's name; control the + // control node's. + byAddress map[string]string + hub string + control string + // silent is every machine whose silence is an open condition. + silent map[string]bool +} + +// judgeNetworks raises and clears every machine's network conditions from every machine's newest +// statement, after one machine's statement was kept. +func judgeNetworks(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, now time.Time) error { + healths, err := inv.Healths(ctx) + if err != nil { + return err + } + overlays, err := inv.Overlays(ctx) + if err != nil { + return err + } + open, err := k.Open(ctx) + if err != nil { + return err + } + f := networkFacts{healths: healths, byAddress: map[string]string{}, control: controlHost(ctx, inv), + silent: map[string]bool{}} + for _, o := range overlays { + if o.Address != "" { + f.byAddress[o.Address] = o.Name + } + if o.Hub { + f.hub = o.Name + } + } + for _, c := range open { + if c.Subject.Scope == conditions.ScopeMachine && c.Kind == "silent" { + f.silent[c.Subject.ID] = true + } + } + var problems []string + said := map[string]bool{} + for _, o := range networkObservations(f) { + said[o.Key()] = true + if _, err := k.Observe(ctx, o); err != nil { + problems = append(problems, err.Error()) + } + } + for _, c := range open { + if !slices.Contains(networkKinds, c.Kind) || said[c.Key] { + continue + } + why := "no machine says it any more" + if c.Subject.Machine != "" { + why = c.Subject.Machine + "'s network no longer says it" + } + if _, err := k.Clear(ctx, c.Key, why); err != nil { + problems = append(problems, err.Error()) + } + } + if len(problems) > 0 { + return fmt.Errorf("%s", strings.Join(problems, "; ")) + } + return nil +} + +// pointed is one machine's failing part that points at another machine. +type pointed struct { + from string + part inventory.NetworkPart +} + +// networkObservations is every network condition the statements say now. Pure. +func networkObservations(f networkFacts) []conditions.Observation { + machines := make([]string, 0, len(f.healths)) + for m := range f.healths { + machines = append(machines, m) + } + sort.Strings(machines) + + unhealthy := func(m string) []inventory.NetworkPart { + h := f.healths[m] + if h.Network == nil { + return nil + } + var out []inventory.NetworkPart + for _, p := range h.Network.Parts { + if p.State == link.StateUnhealthy { + out = append(out, p) + } + } + return out + } + // The machines a part points at, other than its own: the hub, and each mesh resolver by its address. + // An address that is no mesh machine's — the resolver a VPN client wrote in — is the machine's own. + targets := func(m string, p inventory.NetworkPart) ([]string, bool) { + if len(p.Toward) == 0 { + return nil, false + } + var out []string + for _, t := range p.Toward { + x := f.byAddress[t] + if t == link.TowardHub { + x = f.hub + } + if x == "" || x == m { + return nil, false + } + if !slices.Contains(out, x) { + out = append(out, x) + } + } + return out, true + } + + // First pass: what points at whom. + pointing := map[string][]pointed{} + for _, m := range machines { + for _, p := range unhealthy(m) { + if xs, ok := targets(m, p); ok { + for _, x := range xs { + pointing[x] = append(pointing[x], pointed{from: m, part: p}) + } + } + } + } + from := func(x string) []string { + var out []string + for _, pt := range pointing[x] { + if !slices.Contains(out, pt.from) { + out = append(out, pt.from) + } + } + sort.Strings(out) + return out + } + // A machine is down on the record when its silence is open or its own network is unhealthy, or when + // two machines find it unreachable: then what points at it is held there. + down := func(x string) bool { + return f.silent[x] || len(unhealthy(x)) > 0 || len(from(x)) >= 2 + } + + var out []conditions.Observation + saysOwn := map[string]bool{} + for _, m := range machines { + parts := unhealthy(m) + if len(parts) == 0 { + continue + } + var own []inventory.NetworkPart + var rewritten *inventory.NetworkPart + for i, p := range parts { + if p.Part == link.PartResolvConf { + rewritten = &parts[i] + continue + } + if xs, ok := targets(m, p); ok && allDown(xs, down) { + continue // held at the machines it points at + } + own = append(own, p) + } + if rewritten != nil { + out = append(out, rewrittenObservation(m, *rewritten, parts, f)) + // The names failing through what another program wrote are that finding's, said in it. + kept := own[:0] + for _, p := range own { + if p.Part != link.PartNames { + kept = append(kept, p) + } + } + own = kept + } + if len(own) > 0 { + out = append(out, machineNetworkObservation(m, own, f, from(m))) + saysOwn[m] = true + } + } + // Said once, at the machine everybody points at — unless its own network condition already says it + // (listed there), or its silence does. + targetsSorted := make([]string, 0, len(pointing)) + for x := range pointing { + targetsSorted = append(targetsSorted, x) + } + sort.Strings(targetsSorted) + for _, x := range targetsSorted { + if !down(x) || saysOwn[x] || f.silent[x] { + continue + } + out = append(out, unreachableObservation(x, pointing[x], from(x), f)) + } + return out +} + +func allDown(xs []string, down func(string) bool) bool { + for _, x := range xs { + if !down(x) { + return false + } + } + return len(xs) > 0 +} + +// rewrittenObservation is the resolver file rewritten by another program: its own finding, naming the +// writer where the engine could, and what it costs the machine. +func rewrittenObservation(m string, p inventory.NetworkPart, all []inventory.NetworkPart, f networkFacts) conditions.Observation { + writer := "" + if p.Writer != "" { + writer = " (" + p.Writer + ")" + } + cost := "the names through it are not yet judged" + said := []string{p.Part + ": " + p.Said} + for _, q := range all { + if q.Part == link.PartNames { + cost = strings.TrimSuffix(q.Reason, ".") + said = append(said, q.Part+": "+q.Said) + } + } + if cost == "the names through it are not yet judged" { + cost = "the names still resolve through what it wrote" + } + id := m + if p.Owner != "" { + // Named by the module whose file it is: a send that moved that module is what the gate + // holds it on (issue 281's rule — what names a moved module is that module's). + id = m + "." + p.Owner + } + severity := conditions.Warning + if m == f.control { + severity = conditions.Urgent + } + summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the "+ + "node-engine writes it back at its next reconcile, or that program gives it back", m, writer, cost) + if p.Owner != "" { + summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until "+ + "the node-engine writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost) + } + return conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "rewritten", Kind: kindNetworkRewritten, + Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary, + Said: fmt.Sprintf("since %s: %s", p.Since.UTC().Format("2006-01-02 15:04:05 MST"), strings.Join(said, " | "))} +} + +// machineNetworkObservation is what is wrong with a machine's own networking. +func machineNetworkObservation(m string, parts []inventory.NetworkPart, f networkFacts, waiting []string) conditions.Observation { + var words, said []string + severity := conditions.Warning + for _, p := range parts { + words = append(words, p.Reason) + said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, p.Since.UTC().Format("2006-01-02 15:04:05 MST"), p.Said)) + if p.Part == link.PartBus { + severity = conditions.Urgent + } + } + if m == f.control || m == f.hub { + severity = conditions.Urgent + } + summary := fmt.Sprintf("%s's network is not healthy: %s", m, strings.Join(words, "; ")) + if len(waiting) > 0 { + severity = conditions.Urgent + summary += fmt.Sprintf("; %s cannot reach it", strings.Join(waiting, ", ")) + } + return conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Token: "network", Kind: kindMachineNetwork, + Machine: m, Severity: severity, Source: sourceNetwork, Summary: summary, Said: strings.Join(said, " | ")} +} + +// unreachableObservation is one machine others cannot reach, said once there. +func unreachableObservation(x string, pts []pointed, from []string, f networkFacts) conditions.Observation { + var what []string + var said []string + for _, pt := range pts { + w := map[string]string{link.PartTunnel: "the tunnel to it", link.PartBus: "the bus on it", + link.PartNames: "its resolver"}[pt.part.Part] + if w == "" { + w = pt.part.Part + } + if !slices.Contains(what, w) { + what = append(what, w) + } + said = append(said, fmt.Sprintf("%s: %s: %s", pt.from, pt.part.Part, pt.part.Said)) + } + severity := conditions.Warning + if x == f.hub || x == f.control || slices.Contains(what, "the bus on it") { + severity = conditions.Urgent + } + return conditions.Observation{Scope: conditions.ScopeMachine, ID: x, Token: "unreachable", Kind: kindNetworkUnreachable, + Machine: x, Also: from, Severity: severity, Source: sourceNetwork, + Summary: fmt.Sprintf("%s cannot be reached from %s: %s", x, strings.Join(from, ", "), strings.Join(what, ", ")), + Said: strings.Join(said, " | ")} +} + +// networkLines is what `node show` says of a machine's networking. +func networkLines(h inventory.NodeHealth, had bool, now time.Time) []string { + if !had || h.Network == nil { + return []string{" its node-engine does not say how its network is — it is older than that judging (ADR 0241)"} + } + out := []string{fmt.Sprintf(" its network: %s since %s", h.Network.State, h.Network.Since.Local().Format("2006-01-02 15:04"))} + for _, p := range h.Network.Parts { + line := fmt.Sprintf(" %-10s %s", p.State, p.Part) + if p.Reason != "" { + line += " — " + p.Reason + } + if p.Writer != "" { + line += " (" + p.Writer + ")" + } + out = append(out, line) + } + return out +} diff --git a/cmd/mesh-controller/machine_network_test.go b/cmd/mesh-controller/machine_network_test.go new file mode 100644 index 0000000..20bea9d --- /dev/null +++ b/cmd/mesh-controller/machine_network_test.go @@ -0,0 +1,306 @@ +package main + +import ( + "encoding/json" + "os" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A machine says how its network is (novox/hq ADR 0241, "how it is checked"): the resolver file rewritten +// by another program is one finding, naming the writer, with the names it costs said in it; what is the +// machine's own is `machine..network`; what points at another machine that is down is said once, there; +// one machine alone failing toward a healthy one is its own; the control node, the hub and the bus are +// urgent; an engine that says nothing of its network raises nothing; and the gate waits on what is shown to +// be another's. + +func aNetwork(state string, parts ...inventory.NetworkPart) *inventory.NetworkHealth { + for i := range parts { + if parts[i].State == "" { + parts[i].State = link.StateUnhealthy + } + parts[i].Since = h0 + } + return &inventory.NetworkHealth{State: state, Since: h0, Parts: parts} +} + +func netFacts(healths map[string]*inventory.NetworkHealth) networkFacts { + f := networkFacts{healths: map[string]inventory.NodeHealth{}, hub: "anchor", control: "anchor", + byAddress: map[string]string{"10.77.0.1": "anchor", "10.77.0.2": "laptop", "10.77.0.3": "spare"}, + silent: map[string]bool{}} + for m, n := range healths { + f.healths[m] = inventory.NodeHealth{Node: m, Network: n} + } + return f +} + +var ( + rewrittenByVPN = inventory.NetworkPart{Part: link.PartResolvConf, Reason: "the resolver file was rewritten by another program", + Said: "/etc/resolv.conf lists 172.16.5.5 where 10.77.0.1 is declared", Writer: "FortiClient", Owner: "networkmanager"} + namesThroughVPN = inventory.NetworkPart{Part: link.PartNames, Reason: "mesh names do not resolve", + Said: "172.16.5.5 — anchor.internal (IPv4): says no such name", Toward: []string{"172.16.5.5"}} + tunnelDown = inventory.NetworkPart{Part: link.PartTunnel, Reason: "the tunnel to the hub has not handshaken for over five minutes", + Said: "mesh0's newest handshake with the hub was 9m0s ago", Toward: []string{link.TowardHub}} + noRoute = inventory.NetworkPart{Part: link.PartRoute, Reason: "the machine has no default route", Said: "no default route"} +) + +func keysOf(obs []conditions.Observation) []string { + var keys []string + for _, o := range obs { + keys = append(keys, o.Key()) + } + return keys +} + +func TestAResolverFileRewrittenIsOneFindingNamingItsWriterAndWhatItCosts(t *testing.T) { + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, rewrittenByVPN, namesThroughVPN), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" { + t.Fatalf("want one finding, the rewrite, got %v", keys) + } + o := obs[0] + for _, want := range []string{"laptop", "rewritten by another program (FortiClient)", "mesh names do not resolve", + "next reconcile"} { + if !strings.Contains(o.Summary, want) { + t.Errorf("the summary does not say %q: %s", want, o.Summary) + } + } + if strings.Contains(o.Summary, "172.16.") || strings.Contains(o.Summary, "/etc/") { + t.Errorf("an address or a path reached the summary: %s", o.Summary) + } + if !strings.Contains(o.Said, "172.16.5.5") || o.Severity != conditions.Warning || o.Machine != "laptop" { + t.Errorf("the evidence or the severity is wrong: %+v", o) + } +} + +func TestOneMachineFailingTowardAHealthyHubIsItsOwn(t *testing.T) { + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, tunnelDown), + "spare": aNetwork(link.StateHealthy), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" { + t.Fatalf("want the laptop's own, got %v", keys) + } + if obs[0].Severity != conditions.Warning { + t.Fatalf("a laptop's own tunnel is a warning, got %s", obs[0].Severity) + } +} + +func TestTwoMachinesThatCannotReachTheHubAreSaidOnceAtTheHub(t *testing.T) { + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, tunnelDown), + "spare": aNetwork(link.StateUnhealthy, tunnelDown), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.unreachable" { + t.Fatalf("want one condition at the hub, got %v", keys) + } + o := obs[0] + if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop, spare") || len(o.Also) != 2 { + t.Fatalf("the hub's condition is %+v", o) + } +} + +func TestWhatPointsAtASilentHubIsHeldUnderItsSilence(t *testing.T) { + f := netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, tunnelDown)}) + f.silent["anchor"] = true + if obs := networkObservations(f); len(obs) != 0 { + t.Fatalf("the hub's silence says it; got %v", keysOf(obs)) + } +} + +func TestAHubWhoseOwnNetworkIsUnhealthyListsWhoCannotReachIt(t *testing.T) { + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateUnhealthy, noRoute), + "laptop": aNetwork(link.StateUnhealthy, tunnelDown), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.network" { + t.Fatalf("want the hub's own, holding the laptop's, got %v", keys) + } + if o := obs[0]; o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop cannot reach it") { + t.Fatalf("the hub's condition is %+v", o) + } +} + +func TestOneMachineFailingAHealthyMeshResolverIsItsOwnAndTwoAreTheResolvers(t *testing.T) { + silentResolver := inventory.NetworkPart{Part: link.PartNames, Reason: "1 of its 2 resolvers do not answer as the mesh's do", + Said: "10.77.0.3 — no answer within 1s", Toward: []string{"10.77.0.3"}} + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateHealthy), "spare": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, silentResolver), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" { + t.Fatalf("one machine alone: want its own, got %v", keys) + } + obs = networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateUnhealthy, silentResolver), "spare": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, silentResolver), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.spare.unreachable" { + t.Fatalf("two machines: want it said once at the resolver's machine, got %v", keys) + } + if !strings.Contains(obs[0].Summary, "its resolver") { + t.Fatalf("the resolver's machine is said %s", obs[0].Summary) + } +} + +func TestTheControlNodeAndTheBusAreUrgent(t *testing.T) { + f := netFacts(map[string]*inventory.NetworkHealth{"anchor": aNetwork(link.StateUnhealthy, rewrittenByVPN)}) + if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent { + t.Fatalf("the control node's rewritten file: %+v", obs) + } + bus := inventory.NetworkPart{Part: link.PartBus, Reason: "the bus cannot be reached", Said: "no link"} + f = netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, bus, noRoute)}) + if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent { + t.Fatalf("the bus unreachable from the laptop: %+v", obs) + } +} + +func TestAnEngineThatSaysNothingOfItsNetworkRaisesNothing(t *testing.T) { + f := netFacts(map[string]*inventory.NetworkHealth{"laptop": nil, "anchor": aNetwork(link.StateHealthy)}) + if obs := networkObservations(f); len(obs) != 0 { + t.Fatalf("got %v", keysOf(obs)) + } +} + +// Through the store and the keeper: the statement kept, the rewrite raised from it, cleared when the file +// is written back, and node show saying it. +func TestARewrittenResolverFileIsRaisedFromTheStatementAndClearedWhenWrittenBack(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv, k := open.inventory, conditionsFrom + say := func(at time.Time, n *link.NetworkHealth) { + t.Helper() + if err := stateHealth(ctx, inv, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { + t.Fatal(err) + } + } + openKeys := func() []string { + t.Helper() + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + var keys []string + for _, c := range list { + keys = append(keys, c.Key) + } + return keys + } + healthy := &link.NetworkHealth{State: link.StateHealthy, Since: h0, Parts: []link.NetworkPart{ + {Part: link.PartResolvConf, State: link.StateHealthy, Since: h0}}} + rewritten := &link.NetworkHealth{State: link.StateUnhealthy, Since: h0.Add(time.Minute), Parts: []link.NetworkPart{ + {Part: link.PartResolvConf, State: link.StateUnhealthy, Reason: rewrittenByVPN.Reason, Said: rewrittenByVPN.Said, + Writer: "FortiClient", Owner: "networkmanager", Since: h0.Add(time.Minute)}, + {Part: link.PartNames, State: link.StateUnhealthy, Reason: namesThroughVPN.Reason, Said: namesThroughVPN.Said, + Toward: namesThroughVPN.Toward, Since: h0.Add(time.Minute)}}} + + say(h0, healthy) + if keys := openKeys(); len(keys) != 0 { + t.Fatalf("a healthy network raised %v", keys) + } + say(h0.Add(time.Minute), rewritten) + if keys := openKeys(); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" { + t.Fatalf("the rewrite raised %v", keys) + } + kept, had, err := inv.HealthOf(ctx, "laptop") + if err != nil || !had || kept.Network == nil || kept.Network.Parts[0].Writer != "FortiClient" { + t.Fatalf("the statement's network was not kept: %+v %v", kept.Network, err) + } + if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "FortiClient") || + !strings.Contains(lines, "unhealthy resolv-conf") { + t.Fatalf("node show says:\n%s", lines) + } + say(h0.Add(2*time.Minute), healthy) + if keys := openKeys(); len(keys) != 0 { + t.Fatalf("written back, still open: %v", keys) + } + // An engine older than the judging says no network: nothing raised, and node show says it is not known. + say(h0.Add(3*time.Minute), nil) + kept, had, _ = inv.HealthOf(ctx, "laptop") + if keys := openKeys(); len(keys) != 0 || kept.Network != nil { + t.Fatalf("an older engine: %v %+v", keys, kept.Network) + } + if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "older than that judging") { + t.Fatalf("node show says:\n%s", lines) + } +} + +// The gate: a resolver file another program rewrote waits the judging rather than failing it at the +// bound; the same, when the send moved the module whose file it is, is that module's; a machine's own +// network fault holds the machine as a whole, as before. +func TestTheGateWaitsOnARewriteItDidNotMakeAndHoldsTheOwnerOnOneItMoved(t *testing.T) { + since := h0 + rewrite := conditions.Condition{Key: "machine.laptop.networkmanager.rewritten", Kind: kindNetworkRewritten, + Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop.networkmanager", Machine: "laptop"}, + Summary: "the resolver file was rewritten", Raised: since.Add(time.Minute), Source: sourceNetwork} + f := gateFacts{judged: true, open: []conditions.Condition{rewrite}} + if w := aboutTheMachine("laptop", []string{"letta"}, since, f); w.waiting == "" || w.whole != "" || len(w.on) != 0 { + t.Fatalf("a rewrite the send did not make: %+v", w) + } + if w := aboutTheMachine("laptop", []string{"networkmanager", "letta"}, since, f); w.on["networkmanager"] == "" || + w.on["letta"] != "" || w.waiting != "" { + t.Fatalf("a rewrite of the file a moved module owns: %+v", w) + } + own := conditions.Condition{Key: "machine.laptop.network", Kind: kindMachineNetwork, + Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"}, + Summary: "laptop's network is not healthy", Raised: since.Add(time.Minute), Source: sourceNetwork} + if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true, + open: []conditions.Condition{own}}); w.whole == "" || w.waiting != "" { + t.Fatalf("the machine's own network: %+v", w) + } + unreachable := conditions.Condition{Key: "machine.anchor.unreachable", Kind: kindNetworkUnreachable, + Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor", Also: []string{"laptop", "spare"}}, + Summary: "anchor cannot be reached", Raised: since.Add(time.Minute), Source: sourceNetwork} + if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true, + open: []conditions.Condition{unreachable}}); w.waiting == "" || w.whole != "" { + t.Fatalf("a machine that cannot reach the hub: %+v", w) + } +} + +// TestTheDrillsStatementsRaiseAndClearTheRewrite replays the drill of ADR 0241 (mesh-host +// internal/network TestDrill…): what a node-engine said in a throwaway container while its resolver file +// was declared, rewritten as a VPN client rewrites it, and written back — recorded, with the mesh's names +// and addresses replaced by this test mesh's. Healthy raises nothing; the rewrite, on its second look, is +// raised as one finding naming the writer; written back, it clears. +func TestTheDrillsStatementsRaiseAndClearTheRewrite(t *testing.T) { + raw, err := os.ReadFile("testdata/network-drill.json") + if err != nil { + t.Fatal(err) + } + var said []link.Health + if err := json.Unmarshal(raw, &said); err != nil { + t.Fatal(err) + } + open := aMesh(t) + ctx := t.Context() + k := conditionsFrom + var raisedAt []int + for i, h := range said { + if err := stateHealth(ctx, open.inventory, k, "laptop", h, h.At); err != nil { + t.Fatal(err) + } + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + for _, c := range list { + if c.Key != "machine.laptop.networkmanager.rewritten" || !strings.Contains(c.Summary, "(FortiClient)") { + t.Fatalf("statement %d raised %s: %s", i, c.Key, c.Summary) + } + raisedAt = append(raisedAt, i) + } + } + // Five statements: healthy, healthy, one failing look (still healthy), unhealthy, written back. + if len(raisedAt) != 1 || raisedAt[0] != 3 { + t.Fatalf("the rewrite was open after statements %v; want after the fourth alone", raisedAt) + } +} diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index 23ace29..9aa8217 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -83,8 +83,16 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke for module := range unhealthy { streaks[module] = prev.Streaks[module] + 1 } + var network *inventory.NetworkHealth + if h.Network != nil { + network = &inventory.NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: []inventory.NetworkPart{}} + for _, p := range h.Network.Parts { + network.Parts = append(network.Parts, inventory.NetworkPart{Part: p.Part, State: p.State, Reason: p.Reason, + Said: p.Said, Writer: p.Writer, Owner: p.Owner, Toward: p.Toward, Since: p.Since, Streak: p.Streak}) + } + } stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At, - HeardAt: now, Resources: resources, Streaks: streaks}) + HeardAt: now, Resources: resources, Streaks: streaks, Network: network}) if err != nil || !stored { if err == nil { healthRefused.Add(1) @@ -94,7 +102,16 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke if k == nil { return nil } - return judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now) + err = judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now) + // And every machine's network, from every machine's newest statement (ADR 0241): a statement about one + // machine can hold another's finding, or release it. + if nerr := judgeNetworks(ctx, inv, k, now); nerr != nil { + if err == nil { + return nerr + } + return fmt.Errorf("%w; %v", err, nerr) + } + return err } // judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a diff --git a/cmd/mesh-controller/nodes.go b/cmd/mesh-controller/nodes.go index 701b010..8c63a06 100644 --- a/cmd/mesh-controller/nodes.go +++ b/cmd/mesh-controller/nodes.go @@ -546,6 +546,9 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error for _, line := range healthLines(h, had, time.Now()) { fmt.Println(line) } + for _, line := range networkLines(h, had, time.Now()) { + fmt.Println(line) + } } held, err := inv.Profile(ctx, name) diff --git a/cmd/mesh-controller/testdata/network-drill.json b/cmd/mesh-controller/testdata/network-drill.json new file mode 100644 index 0000000..edf7321 --- /dev/null +++ b/cmd/mesh-controller/testdata/network-drill.json @@ -0,0 +1,170 @@ +[ + { + "contract": 2, + "at": "2026-10-07T16:48:58.691388404Z", + "resources": [], + "network": { + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z", + "parts": [ + { + "part": "resolv-conf", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + }, + { + "part": "names", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + }, + { + "part": "bus", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + }, + { + "part": "route", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + } + ] + } + }, + { + "contract": 2, + "at": "2026-10-07T16:49:28.691388404Z", + "resources": [], + "network": { + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z", + "parts": [ + { + "part": "resolv-conf", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + }, + { + "part": "names", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + }, + { + "part": "bus", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + }, + { + "part": "route", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + } + ] + } + }, + { + "contract": 2, + "at": "2026-10-07T16:49:58.691388404Z", + "resources": [], + "network": { + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z", + "parts": [ + { + "part": "resolv-conf", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z", + "streak": 1 + }, + { + "part": "names", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z", + "streak": 1 + }, + { + "part": "bus", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + }, + { + "part": "route", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + } + ] + } + }, + { + "contract": 2, + "at": "2026-10-07T16:50:28.691388404Z", + "resources": [], + "network": { + "state": "unhealthy", + "since": "2026-10-07T16:50:28.691388404Z", + "parts": [ + { + "part": "resolv-conf", + "state": "unhealthy", + "reason": "the resolver file was rewritten by another program", + "said": "/etc/resolv.conf differs from what networkmanager declares: it lists 192.0.2.53 where 10.77.0.2, 10.77.0.1 is declared; changed 2026-10-07T16:48:28Z; its own header names FortiClient", + "writer": "FortiClient", + "owner": "networkmanager", + "since": "2026-10-07T16:50:28.691388404Z", + "streak": 2 + }, + { + "part": "names", + "state": "unhealthy", + "reason": "neither mesh names nor public names resolve", + "said": "within 1s: 192.0.2.53 — anchor.internal (IPv4): no answer within 1s; anchor.internal (IPv6): no answer within 1s; example.com (IPv4): no answer within 1s", + "toward": [ + "192.0.2.53" + ], + "since": "2026-10-07T16:50:28.691388404Z", + "streak": 2 + }, + { + "part": "bus", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + }, + { + "part": "route", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + } + ] + } + }, + { + "contract": 2, + "at": "2026-10-07T16:50:58.691388404Z", + "resources": [], + "network": { + "state": "healthy", + "since": "2026-10-07T16:50:58.691388404Z", + "parts": [ + { + "part": "resolv-conf", + "state": "healthy", + "since": "2026-10-07T16:50:58.691388404Z" + }, + { + "part": "names", + "state": "healthy", + "since": "2026-10-07T16:50:58.691388404Z" + }, + { + "part": "bus", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + }, + { + "part": "route", + "state": "healthy", + "since": "2026-10-07T16:48:58.691388404Z" + } + ] + } + } +] \ No newline at end of file diff --git a/internal/inventory/health.go b/internal/inventory/health.go index a350080..fe31d5c 100644 --- a/internal/inventory/health.go +++ b/internal/inventory/health.go @@ -40,6 +40,29 @@ type NodeHealth struct { Resources []ResourceHealth // Streaks is, per module, how many statements in a row said a resource of it was unhealthy. Streaks map[string]int + // Network is the machine's own networking as its engine said it (novox/hq ADR 0241); nil from an + // engine older than that judging. + Network *NetworkHealth +} + +// NetworkHealth is a machine's networking as its engine said it (ADR 0241). +type NetworkHealth struct { + State string `json:"state"` + Since time.Time `json:"since"` + Parts []NetworkPart `json:"parts"` +} + +// NetworkPart is one part of it: resolv-conf, names, tunnel, bus or route. +type NetworkPart struct { + Part string `json:"part"` + State string `json:"state"` + Reason string `json:"reason,omitempty"` + Said string `json:"said,omitempty"` + Writer string `json:"writer,omitempty"` + Owner string `json:"owner,omitempty"` + Toward []string `json:"toward,omitempty"` + Since time.Time `json:"since"` + Streak int `json:"streak,omitempty"` } // HealthOf is a machine's newest statement; false when its node-engine has never stated one. @@ -60,7 +83,7 @@ func (i *Inventory) Healths(ctx context.Context) (map[string]NodeHealth, error) func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHealth, error) { rows, err := i.store.Pool().Query(ctx, - `select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks + `select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks, h.network from node_health h join node n on n.id = h.node where $1 = '' or n.name = $1`, only) if err != nil { @@ -70,8 +93,8 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe out := map[string]NodeHealth{} for rows.Next() { var h NodeHealth - var resources, streaks []byte - if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks); err != nil { + var resources, streaks, network []byte + if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks, &network); err != nil { return nil, err } if err := json.Unmarshal(resources, &h.Resources); err != nil { @@ -80,6 +103,11 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe if err := json.Unmarshal(streaks, &h.Streaks); err != nil { return nil, fmt.Errorf("%s's health cannot be read: %w", h.Node, err) } + if len(network) > 0 { + if err := json.Unmarshal(network, &h.Network); err != nil { + return nil, fmt.Errorf("%s's network health cannot be read: %w", h.Node, err) + } + } out[h.Node] = h } return out, rows.Err() @@ -102,18 +130,25 @@ func (i *Inventory) RecordHealth(ctx context.Context, h NodeHealth) (bool, error if err != nil { return false, err } + var network []byte + if h.Network != nil { + if network, err = json.Marshal(h.Network); err != nil { + return false, err + } + } heard := h.HeardAt if heard.IsZero() { heard = time.Now() } var node string err = i.store.Pool().QueryRow(ctx, - `insert into node_health (node, contract, said_at, heard_at, resources, streaks) - select id, $2, $3, $4, $5, $6 from node where name = $1 + `insert into node_health (node, contract, said_at, heard_at, resources, streaks, network) + select id, $2, $3, $4, $5, $6, $7 from node where name = $1 on conflict (node) do update set contract = excluded.contract, said_at = excluded.said_at, - heard_at = excluded.heard_at, resources = excluded.resources, streaks = excluded.streaks + heard_at = excluded.heard_at, resources = excluded.resources, streaks = excluded.streaks, + network = excluded.network where node_health.said_at <= excluded.said_at - returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks).Scan(&node) + returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks, network).Scan(&node) if errors.Is(err, pgx.ErrNoRows) { if _, nerr := i.NodeByName(ctx, h.Node); nerr != nil { return false, nerr diff --git a/internal/inventory/migrations/0077-a-machine-says-how-its-network-is.sql b/internal/inventory/migrations/0077-a-machine-says-how-its-network-is.sql new file mode 100644 index 0000000..2da375f --- /dev/null +++ b/internal/inventory/migrations/0077-a-machine-says-how-its-network-is.sql @@ -0,0 +1,10 @@ +-- A machine says how its network is (novox/hq ADR 0241, which extends ADR 0240 from what a module runs to +-- the machine it runs on). +-- +-- Beside the state of every long-running resource, each machine's node-engine states its own networking: +-- the resolver file the uplink holder declared and who rewrote it, the names through every resolver it +-- lists, the tunnel's handshake with the hub, the bus and the default route — the worst of them, since +-- when, and each part. Kept with the machine's newest statement, replaced with it, so `node show`, the +-- gate and a controller started again read the same word. Null for a machine whose node-engine is older +-- than this judging: its network is not known — never healthy, never a reason to raise anything. +alter table node_health add column network jsonb; diff --git a/internal/link/protocol.go b/internal/link/protocol.go index e7767f7..ad27bec 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -428,6 +428,45 @@ type Health struct { // At is when the engine looked, on the machine's clock: the order of its statements. At time.Time `json:"at"` Resources []ResourceHealth `json:"resources"` + // Network is the machine's own networking, judged by its engine (novox/hq ADR 0241); nil from an engine + // older than that judging, which is "not known", never healthy. + Network *NetworkHealth `json:"network,omitempty"` +} + +// NetworkHealth is a machine's networking in one statement (ADR 0241): the worst of its parts, since +// when, and each part. The node-engine's own (mesh-host internal/link NetworkHealth). +type NetworkHealth struct { + State string `json:"state"` + Since time.Time `json:"since"` + Parts []NetworkPart `json:"parts"` +} + +// The parts of a machine's networking its engine judges (ADR 0241). +const ( + PartResolvConf = "resolv-conf" + PartNames = "names" + PartTunnel = "tunnel" + PartBus = "bus" + PartRoute = "route" + // TowardHub is what a part failing toward the hub names in Toward. + TowardHub = "hub" +) + +// NetworkPart is one part, as the engine judged it on its second look. +type NetworkPart struct { + Part string `json:"part"` + State string `json:"state"` + // Reason is in words with no address, path or domain; Said is the detail, kept as evidence. + Reason string `json:"reason,omitempty"` + Said string `json:"said,omitempty"` + // Writer is the program that rewrote the resolver file, when the engine could name it; Owner the + // module whose file it is. + Writer string `json:"writer,omitempty"` + Owner string `json:"owner,omitempty"` + // Toward is what a failure points at: "hub", or each resolver's address that failed. + Toward []string `json:"toward,omitempty"` + Since time.Time `json:"since"` + Streak int `json:"streak,omitempty"` } // The states a resource is said in (ADR 0240 §4). diff --git a/internal/link/protocol_test.go b/internal/link/protocol_test.go index 4f8a412..cc2120a 100644 --- a/internal/link/protocol_test.go +++ b/internal/link/protocol_test.go @@ -33,6 +33,13 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { []string{"module", "resource", "kind", "target", "state", "reason", "since", "streak", "restarts", "check", "needs"}}, {HealthSaid{Node: "n"}, []string{"node", "health"}}, + // novox/hq ADR 0241: the machine's own networking, beside its resources. + {Health{Contract: ReadinessContract, Resources: []ResourceHealth{}, Network: &NetworkHealth{State: "unhealthy", + Parts: []NetworkPart{}}}, []string{"contract", "at", "resources", "network"}}, + {NetworkHealth{State: "unhealthy", Parts: []NetworkPart{}}, []string{"state", "since", "parts"}}, + {NetworkPart{Part: "resolv-conf", State: "unhealthy", Reason: "r", Said: "s", Writer: "w", Owner: "o", + Toward: []string{"hub"}, Streak: 2}, []string{"part", "state", "reason", "said", "writer", "owner", "toward", + "since", "streak"}}, } { raw, err := json.Marshal(c.value) if err != nil {