diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index 62b84a8..660ea74 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -321,6 +321,25 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }() }) } + // Each node's own health report (the node-engine's state health): read before the findings are reconciled when the + // bucket is there, bound once it appears when it is not; each change is heard on the watch, never polled. + healthEach := func(e jetstream.KeyValueEntry) { w.NodeHealth(e.Key(), e.Value(), opOf(e)) } + var healthWatch jetstream.KeyWatcher + if kv, err := js.KeyValue(ctx, delivery.NodeHealthBucket); err == nil { + if healthWatch, err = settle(ctx, kv, healthEach); err != nil { + process.NotWatched(logger.Printf, delivery.NodeHealthBucket, err, "no node's own health is said until the watchman starts again") + } + } else { + process.AwaitBucket(ctx, js, conn, delivery.NodeHealthBucket, "no node's own health is said", + logger.Printf).Ready(func(kv jetstream.KeyValue) { + hw, err := settle(ctx, kv, healthEach) + if err != nil { + process.NotWatched(logger.Printf, delivery.NodeHealthBucket, err, "no node's own health is said until the watchman starts again") + return + } + go func() { _ = follow(ctx, hw, healthEach) }() + }) + } dead := &deadKeeper{ctx: ctx, js: js, w: w, logf: logger.Printf} w.SetRecount(dead.count) dead.count() @@ -332,7 +351,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } - errs := make(chan error, 8) + errs := make(chan error, 16) + if healthWatch != nil { + go func() { errs <- follow(ctx, healthWatch, healthEach) }() + } if provWatch != nil { go func() { errs <- follow(ctx, provWatch, func(e jetstream.KeyValueEntry) { w.Provision(e.Key(), e.Value(), opOf(e)) }) diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index 8842356..db64d6f 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-watchman go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215805-94d0de5c19b6 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-watchman/go.sum b/modules/mesh-delivery-watchman/go.sum index b279ec6..d016872 100644 --- a/modules/mesh-delivery-watchman/go.sum +++ b/modules/mesh-delivery-watchman/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368 h1:LaBxsyU3HeTIPKYmvpJuw+Wm2plpf6O7qHO91O+mAi0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215805-94d0de5c19b6 h1:eQIsSB3Z72MNjirW3TbKbvFgkhwjCAgOAqWd0U4ldCE= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215805-94d0de5c19b6/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index 182c5ca..4fbf380 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -173,6 +173,59 @@ func TestTheWatchmansFindings(t *testing.T) { w.Provision("ace.web.oidc", enc(provisions.Waits, "x"), Put) w.Provision("ace.web.oidc", nil, Delete) }, map[string]string{}}, + {"774: a node's own report: its network, the resolver file rewritten, failed units, a relogin, a directory used as found", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + report := func(h delivery.NodeHealth) []byte { + b, err := delivery.EncodeNodeHealth(h) + if err != nil { + t.Fatal(err) + } + return b + } + w.Have("novox.mesh-controller", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) // the control node + w.NodeHealth("novox", report(delivery.NodeHealth{Node: "novox", Network: &delivery.NetworkHealth{State: "unhealthy", Parts: []delivery.NetworkPart{ + {Part: "resolv-conf", State: "unhealthy", Writer: "NetworkManager", Owner: "systemd-resolved", Said: "rewritten"}, + {Part: "names", State: "unhealthy", Reason: "names do not resolve.", Said: "x"}, + {Part: "route", State: "unhealthy", Reason: "no default route", Said: "y"}}}, + Units: &delivery.UnitsHealth{State: "degraded", Failed: []delivery.FailedUnit{{Unit: "cups.service", Scope: "system", Result: "exit-code"}}}, + Relogin: []delivery.NodeAccount{{Account: "jochen", Module: "docker", Reason: "a group"}}, + Found: []delivery.NodeDirectory{{Module: "photos", Resource: "data", Path: "/srv/photos", Reason: "owner differs"}}}), Put) + for key, sev := range map[string]string{"machine.novox.network": Urgent, "machine.novox.systemd-resolved.rewritten": Urgent, + "machine.novox.units": Warning, "module.docker.novox.relogin-needed": Warning, "module.photos.novox.directory-used-as-found": Warning} { + if o.findings[key].Severity != sev { + t.Errorf("%s = %+v", key, o.findings[key]) + } + } + if f := o.findings["machine.novox.network"]; strings.Contains(f.Summary, "names") { + t.Errorf("the names failing through the rewritten file are said twice: %s", f.Summary) + } + w.NodeHealth("novox", report(delivery.NodeHealth{Node: "novox"}), Put) // all well again + }, map[string]string{}}, + {"774: a node two nodes' tunnels no longer see is unreachable, said once there; not while it is silent", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + for node, key := range map[string]string{"ace": "kA", "g14": "kG", "lap": "kL"} { + b, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node, TunnelKey: key}}) + w.Have(node+".node", b, Put) + } + peers := func(from string, seen bool) []byte { + b, _ := delivery.EncodeNodeHealth(delivery.NodeHealth{Node: from, Peers: []delivery.PeerSeen{{Key: "kL", Seen: seen}}}) + return b + } + w.NodeHealth("ace", peers("ace", false), Put) + if len(o.findings) != 0 { + t.Fatalf("one node alone said it: %v", o.findings) + } + w.NodeHealth("g14", peers("g14", false), Put) + if f := o.findings["machine.lap.unreachable"]; len(f.Also) != 2 { + t.Fatalf("findings = %v", o.findings) + } + w.Connect("node.lap") + w.Disconnect("node.lap") + expire(w, c, pulseKeyOf(t, presenceNS, "lap"), 120*time.Second) + if _, ok := o.findings["machine.lap.unreachable"]; ok { + t.Error("a silent node is also said unreachable") + } + w.Connect("node.lap") + w.NodeHealth("g14", peers("g14", true), Put) + }, map[string]string{}}, {"WM3: a have removed ends its findings", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "x"), Put) w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "x"), Put) diff --git a/modules/mesh-delivery-watchman/internal/watch/mirror.go b/modules/mesh-delivery-watchman/internal/watch/mirror.go index 4eee236..fe9f262 100644 --- a/modules/mesh-delivery-watchman/internal/watch/mirror.go +++ b/modules/mesh-delivery-watchman/internal/watch/mirror.go @@ -106,11 +106,22 @@ func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) { // fingerprint of the node's want for the module ends the want's pulse. A node's own key and a key deleted are left. func (w *Watchman) Have(key string, value []byte, op Op) { node, module, isNode, err := delivery.ParseHaveKey(key) - if err != nil || isNode { + if err != nil { return } w.mu.Lock() defer w.mu.Unlock() + if isNode { + // The node's own record: its tunnel key, whose peers' sight of it is network-unreachable. + var record *delivery.NodeRecord + if op == Put { + if hn, err := delivery.DecodeHaveNode(value); err == nil { + record = hn.Record + } + } + w.tunnelKey(node, record) + return + } switch op { case Expired: return diff --git a/modules/mesh-delivery-watchman/internal/watch/nodehealth.go b/modules/mesh-delivery-watchman/internal/watch/nodehealth.go new file mode 100644 index 0000000..7cb6f91 --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/nodehealth.go @@ -0,0 +1,358 @@ +package watch + +import ( + "fmt" + "maps" + "slices" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// A node's own health report (novox/hq ADR 0241, issue 315, the operator's alert table of 2026-10-29): each node's engine +// writes `mesh-host_health.` when what it says changes, and the watchman raises from it what the controller raised +// from the passing health message, under the same keys and words (mesh-control cmd/mesh-controller machine_network.go, +// machine_units.go, module_health.go at e04846e9): +// - machine-network `machine..network`: the machine's own unhealthy parts; urgent on the control node or for the bus; +// - network-rewritten `machine.[.].rewritten`: the resolver file rewritten by another program; +// - network-unreachable `machine..unreachable`: two or more nodes' tunnels no longer see it (their peers' handshake), +// said once there, never while its own network or its silence says it; +// - machine-units `machine..units`: failed units no module places; +// - relogin-needed `module...relogin-needed` and directory-used-as-found +// `module...directory-used-as-found`. +// +// What the controller held at another machine by the addresses its parts point at (Toward) needs each node's private +// address and the hub, which no report carries: a part pointing elsewhere is the machine's own here, and the machine all +// point at is said by its peers instead. + +// The kinds a node's health report raises. +const ( + KindMachineNetwork = "machine-network" + KindNetworkRewritten = "network-rewritten" + KindNetworkUnreachable = "network-unreachable" + KindMachineUnits = "machine-units" + KindReloginNeeded = "relogin-needed" + KindUsedAsFound = "directory-used-as-found" +) + +// The parts and states a report names (mesh-control internal/link). +const ( + partResolvConf = "resolv-conf" + partNames = "names" + partTunnel = "tunnel" + partBus = "bus" + stateUnhealthy = "unhealthy" +) + +func stampSec(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05 MST") } + +// NodeHealth is a change of a node's report: written, or deleted (value nil). +func (w *Watchman) NodeHealth(node string, value []byte, op Op) { + w.mu.Lock() + defer w.mu.Unlock() + if op == Put { + h, err := delivery.DecodeNodeHealth(value) + if err != nil { + w.say("the health report of %s cannot be read, and is left: %v", node, err) + return + } + w.health[node] = h + } else { + delete(w.health, node) + } + w.nodeFindings(node) + w.unreachable() +} + +// TunnelKey is a node's own tunnel key, from its record in its have: whose peers see it is read by it. +func (w *Watchman) tunnelKey(node string, record *delivery.NodeRecord) { + key := "" + if record != nil { + key = record.TunnelKey + } + if w.tunnels[node] == key { + return + } + if key == "" { + delete(w.tunnels, node) + } else { + w.tunnels[node] = key + } + w.unreachable() +} + +// nodeFindings writes and deletes one node's own findings from its report. +func (w *Watchman) nodeFindings(node string) { + h, ok := w.health[node] + want := map[string]bool{} + if ok { + for _, f := range w.networkOf(node, h) { + want[f.Key()] = true + w.find(f) + } + if f, raise := unitsOf(node, h.Units); raise { + want[f.Key()] = true + w.find(f) + } + for _, f := range reloginOf(node, h.Relogin) { + want[f.Key()] = true + w.find(f) + } + for _, f := range foundOf(node, h.Found) { + want[f.Key()] = true + w.find(f) + } + } + for key := range w.findings { + if !want[key] && w.ownsHealthKey(key, node) { + w.unfind(key) + } + } +} + +// ownsHealthKey says a finding key is one of node's own report findings. +func (w *Watchman) ownsHealthKey(key, node string) bool { + f, err := DecodeFinding(w.findings[key]) + if err != nil || f.Machine != node || f.Source != "health-report" { + return false + } + return f.Kind != KindNetworkUnreachable +} + +// networkOf is a node's own network findings: the resolver file rewritten, and the rest of its unhealthy parts. +func (w *Watchman) networkOf(m string, h delivery.NodeHealth) []Finding { + if h.Network == nil { + return nil + } + var parts []delivery.NetworkPart + for _, p := range h.Network.Parts { + if p.State == stateUnhealthy { + parts = append(parts, p) + } + } + var out []Finding + var own []delivery.NetworkPart + var rewritten *delivery.NetworkPart + for i, p := range parts { + if p.Part == partResolvConf { + rewritten = &parts[i] + continue + } + own = append(own, p) + } + if rewritten != nil { + out = append(out, w.rewrittenOf(m, *rewritten, parts)) + kept := own[:0] + for _, p := range own { + if p.Part != partNames { + kept = append(kept, p) + } + } + own = kept + } + if len(own) > 0 { + var words, said []string + severity := Warning + for _, p := range own { + words = append(words, p.Reason) + said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, stampSec(p.Since), p.Said)) + if p.Part == partBus { + severity = Urgent + } + } + if w.control[m] { + severity = Urgent + } + out = append(out, Finding{Scope: ScopeMachine, ID: m, Token: "network", Kind: KindMachineNetwork, Machine: m, + Severity: severity, Source: "health-report", Since: h.Network.Since, + Summary: fmt.Sprintf("%s's network is not healthy: %s", m, strings.Join(words, "; ")), Said: strings.Join(said, " | ")}) + } + return out +} + +func (w *Watchman) rewrittenOf(m string, p delivery.NetworkPart, all []delivery.NetworkPart) Finding { + writer := "" + if p.Writer != "" { + writer = " (" + p.Writer + ")" + } + cost := "the names still resolve through what it wrote" + said := []string{p.Part + ": " + p.Said} + for _, q := range all { + if q.Part == partNames { + cost = strings.TrimSuffix(q.Reason, ".") + said = append(said, q.Part+": "+q.Said) + } + } + id := m + summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the node-engine writes it "+ + "back at its next reconcile, or that program gives it back", m, writer, cost) + if p.Owner != "" { + id = m + "." + p.Owner + summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until the node-engine "+ + "writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost) + } + severity := Warning + if w.control[m] { + severity = Urgent + } + return Finding{Scope: ScopeMachine, ID: id, Token: "rewritten", Kind: KindNetworkRewritten, Machine: m, Severity: severity, + Source: "health-report", Since: p.Since, Summary: summary, + Said: fmt.Sprintf("since %s: %s", stampSec(p.Since), strings.Join(said, " | "))} +} + +// unitsOf is a machine's failed units no module places, one finding. +func unitsOf(node string, u *delivery.UnitsHealth) (Finding, bool) { + if u == nil || len(u.Failed) == 0 { + return Finding{}, false + } + var names, said []string + since := u.Failed[0].Since + for _, f := range u.Failed { + where := "" + if f.Scope == "user" { + where = " (the account's own manager)" + } + names = append(names, f.Unit+where) + scope, result := f.Scope, f.Result + if scope == "" { + scope = "system" + } + if result == "" { + result = "no reason said" + } + line := fmt.Sprintf("%s in the %s manager: %s", f.Unit, scope, result) + if f.Load != "" && f.Load != "loaded" { + line += ", its unit " + f.Load + } + if f.Resource != "" { + line += ", named by the mesh's own " + f.Resource + } + said = append(said, line+", since "+stampSec(f.Since)) + if f.Since.Before(since) { + since = f.Since + } + } + failed := "1 failed service" + if len(u.Failed) > 1 { + failed = fmt.Sprintf("%d failed services", len(u.Failed)) + } + return Finding{Scope: ScopeMachine, ID: node, Token: "units", Kind: KindMachineUnits, Machine: node, Severity: Warning, + Source: "health-report", Since: since, + Summary: fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. Each is the machine's own: "+ + "mend or remove it there, or have a module place it", node, len(u.Failed), strings.Join(names, ", ")), + Said: strings.Join(said, "; "), + Headline: strings.ToUpper(failed[:1]) + failed[1:] + " on " + node, + Needs: fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node), + Resolved: "No failed services on " + node + " any more"}, true +} + +// reloginOf is one finding per module whose accounts wait for a new login. +func reloginOf(node string, accounts []delivery.NodeAccount) []Finding { + by := map[string][]delivery.NodeAccount{} + for _, a := range accounts { + by[a.Module] = append(by[a.Module], a) + } + var out []Finding + for _, module := range slices.Sorted(maps.Keys(by)) { + var names []string + since := by[module][0].Since + for _, a := range by[module] { + names = append(names, a.Account) + if a.Since.Before(since) { + since = a.Since + } + } + sort.Strings(names) + said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose login began "+ + "before it was; log out of %[1]s completely and log in again, or restart it", node, orModule(module), strings.Join(names, ", ")) + out = append(out, Finding{Scope: ScopeModule, ID: orModule(module) + "." + node, Kind: KindReloginNeeded, Machine: node, + Severity: Warning, Source: "health-report", Since: since, Summary: said, Said: said}) + } + return out +} + +// foundOf is one finding per module with a directory used as found. +func foundOf(node string, dirs []delivery.NodeDirectory) []Finding { + by := map[string][]delivery.NodeDirectory{} + for _, d := range dirs { + by[d.Module] = append(by[d.Module], d) + } + var out []Finding + for _, module := range slices.Sorted(maps.Keys(by)) { + var ids, why []string + since := by[module][0].Since + for _, d := range by[module] { + ids = append(ids, d.Resource) + why = append(why, d.Reason) + if d.Since.Before(since) { + since = d.Since + } + } + m := orModule(module) + said := fmt.Sprintf("%s on %s uses %s as found: %s", m, node, strings.Join(ids, ", "), strings.Join(why, "; ")) + out = append(out, Finding{Scope: ScopeModule, ID: m + "." + node, Kind: KindUsedAsFound, Machine: node, Severity: Warning, + Source: "health-report", Since: since, Summary: said, Said: said, + Headline: fmt.Sprintf("%s waits for a directory on %s", m, node), + Explanation: fmt.Sprintf("Needs you: hand the directory over from the control-node, as the operator; the details name it "+ + "and the line to type. A directory of %s was already on %s, with another owner or mode than %s declares.", m, node, m), + Needs: "hand the directory over from the control-node, as the operator; the details name it and the line to type.", + Resolved: fmt.Sprintf("%s's directory on %s is the mesh's", m, node)}) + } + return out +} + +func orModule(m string) string { + if m == "" { + return "_" + } + return m +} + +// unreachable is network-unreachable across every report: a node whose tunnel key two or more other nodes' peers list +// as not seen, said once there, unless its own network finding or its silence says it. +func (w *Watchman) unreachable() { + byKey := map[string]string{} + for node, key := range w.tunnels { + byKey[key] = node + } + notSeen := map[string][]string{} + for from, h := range w.health { + for _, p := range h.Peers { + x, ok := byKey[p.Key] + if !ok || x == from || p.Seen { + continue + } + notSeen[x] = append(notSeen[x], from) + } + } + want := map[string]bool{} + for x, from := range notSeen { + if len(from) < 2 { + continue + } + if _, own := w.findings[FindingKey(ScopeMachine, x, "network")]; own { + continue + } + if _, silent := w.findings[FindingKey(ScopeMachine, x, KindSilent)]; silent { + continue + } + sort.Strings(from) + severity := Warning + if w.control[x] { + severity = Urgent + } + f := Finding{Scope: ScopeMachine, ID: x, Token: "unreachable", Kind: KindNetworkUnreachable, Machine: x, Also: from, + Severity: severity, Source: "health-report", + Summary: fmt.Sprintf("%s cannot be reached from %s: the tunnel to it", x, strings.Join(from, ", ")), + Said: fmt.Sprintf("%s: tunnel: no handshake with %s within five minutes", strings.Join(from, ", "), x)} + want[f.Key()] = true + w.find(f) + } + for key := range w.findings { + if strings.HasSuffix(key, ".unreachable") && strings.HasPrefix(key, ScopeMachine+".") && !want[key] { + w.unfind(key) + } + } +} diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go index 45c2881..217f896 100644 --- a/modules/mesh-delivery-watchman/internal/watch/signals.go +++ b/modules/mesh-delivery-watchman/internal/watch/signals.go @@ -240,6 +240,7 @@ func (w *Watchman) stuckFinding(sk string, s delivery.Stuck) { if s.Bucket == delivery.PresenceBucket { if node, part, _, err := delivery.ParsePresenceKey(s.Key); err == nil && part == "" && node != "" { key := FindingKey(ScopeMachine, node, KindSilent) + defer w.unreachable() // a silent machine is said by its silence, never also as unreachable if s.Moved || w.away[node] { w.unfind(key) w.unsignal("S1", node) diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index 84e1ef0..cc4dcc3 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -86,6 +86,9 @@ type Watchman struct { recount func() // waiting is, by `.`, each of its provisions in waits, with its words. waiting map[string]map[string]string + // health is each node's own health report, and tunnels each node's tunnel key from its record. + health map[string]delivery.NodeHealth + tunnels map[string]string // stepBound is how long a build step may run once a node took the ask; a test shortens it. stepBound time.Duration } @@ -96,7 +99,8 @@ func New(out Out, now func() time.Time) *Watchman { pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, bound: delivery.PulseMoveBound, stepBound: StepBound, findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, away: map[string]bool{}, haveState: map[string]delivery.Have{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}, - control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}} + control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}, + health: map[string]delivery.NodeHealth{}, tunnels: map[string]string{}} } // Logf sets where a write that failed is said; it is silent until set. diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index 21a1ad5..a57ddfe 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -91,7 +91,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 reads := slices.Clone(m.Reads) slices.Sort(reads) if !reflect.DeepEqual(reads, []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-controller.lease", - "mesh-delivery-assigner.wants", "mesh-provisioner.provisions", "node-engine.have"}) { + "mesh-delivery-assigner.wants", "mesh-host.health", "mesh-provisioner.provisions", "node-engine.have"}) { t.Errorf("reads = %v", reads) } // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index 291908c..fcc3077 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -21,6 +21,7 @@ "mesh-build-node.steps", "mesh-controller.lease", "mesh-delivery-assigner.wants", + "mesh-host.health", "mesh-provisioner.provisions", "node-engine.have" ],