diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 97bb99a..c73690b 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -79,6 +79,8 @@ func engineBeyond(user, g string) bool { strings.Contains(g, "KV_"+EngineActsBucket) || strings.HasPrefix(g, "$KV."+EngineActsBucket+"."+node+".") || // And its node's declared packages (novox/hq ADR 0565 clause 6). strings.Contains(g, "KV_"+EnginePackagesBucket) || g == "$KV."+EnginePackagesBucket+"."+node || + // And its own health report (issue 774). + strings.Contains(g, "KV_"+delivery.NodeHealthBucket) || g == "$KV."+delivery.NodeHealthBucket+"."+node || // And the provisions of its node's modules, read (novox/hq ADR 0570 clause 6). strings.Contains(g, "KV_"+provisions.ProvisionsBucket) } @@ -601,7 +603,7 @@ func healthChecks(t *testing.T) { Reads: []string{"mesh-self-check.findings", "mesh-healers.attempts", ControllerConditions}}, {Module: "mesh-self-check", State: []Bucket{bucket("mesh-self-check", "findings")}}, {Module: "mesh-healers", State: []Bucket{bucket("mesh-healers", "findings"), bucket("mesh-healers", "attempts"), bucket("mesh-healers", "would")}}, - {Module: WatchmanModule, State: []Bucket{bucket(WatchmanModule, "findings")}}, + {Module: WatchmanModule, State: []Bucket{bucket(WatchmanModule, "findings")}, Reads: []string{delivery.NodeHealthRead}}, {Module: "mesh-hand-acts", State: []Bucket{bucket("mesh-hand-acts", "warrants")}, Logs: []Log{{Module: "mesh-hand-acts", Name: "acts"}}}, // ADR 0570 clauses 1, 8 (issue 1954 L6-P5): mesh-provisioner keeps provisions and plans; the vault and a provider // read them. @@ -675,6 +677,14 @@ func healthChecks(t *testing.T) { MayPublish(perms, "$KV.mesh-provisioner_provisions.a.keycloak.db")) { t.Errorf("node %s reads the provisions %v", u.Node, perms.Publish) } + // Issue 774: a node's engine writes its own health report's key and no other node's. + if u.Kind == KindNode && (!MayPublish(perms, "$KV.mesh-host_health.a") || MayPublish(perms, "$KV.mesh-host_health.b")) { + t.Errorf("node %s writes the health reports %v", u.Node, perms.Publish) + } + if u.Kind == KindModule && u.Module == WatchmanModule && (!MayPublish(perms, "$JS.API.DIRECT.GET.KV_mesh-host_health.$KV.mesh-host_health.b") || + MayPublish(perms, "$KV.mesh-host_health.b")) { + t.Errorf("the watchman reads the health reports %v, %v", MayPublish(perms, "$JS.API.DIRECT.GET.KV_mesh-host_health.$KV.mesh-host_health.b"), perms.Publish) + } if u.Kind == KindNode && (!MayPublish(perms, "$KV.mesh-host_packages.a") || MayPublish(perms, "$KV.mesh-host_packages.b")) { t.Errorf("node %s writes the packages keys %v", u.Node, perms.Publish) } diff --git a/go/buscompose/engine.go b/go/buscompose/engine.go index 4681729..bc504f3 100644 --- a/go/buscompose/engine.go +++ b/go/buscompose/engine.go @@ -58,6 +58,12 @@ func engineGrants(p Principal) (pub, sub []string) { if safeSubject.MatchString(p.Node) { pub = append(pub, readGrants(provisions.ProvisionsBucket, p.Node+".>")...) } + // And its node's own health report, its key `` alone (issue 774, the operator's alert table): what the watchman + // reads for the machine's alerts. + if safeSubject.MatchString(p.Node) { + pub = append(pub, append([]string{"$KV." + delivery.NodeHealthBucket + "." + p.Node}, + readGrants(delivery.NodeHealthBucket, p.Node)...)...) + } // And its node's declared packages, its own key alone (ADR 0565 clause 6). if safeSubject.MatchString(p.Node) { pub = append(pub, EnginePackagesGrants(p.Node)...) diff --git a/go/buscompose/writers.go b/go/buscompose/writers.go index 77abcd2..6da9693 100644 --- a/go/buscompose/writers.go +++ b/go/buscompose/writers.go @@ -248,6 +248,10 @@ var WritersTable = []WriterRow{ Others: "read through `reads` as node-engine.have", Subjects: kvOf(delivery.HaveBucket), Writes: ownNodeKeys}, // The acts a node's engine was asked through its node seat, and its own asks (novox/hq ADR 0563 §2, §3): the // node-engine's declared state, created by the grants part, written by each node's engine under its own keys. + // A node's own health report (issue 774): its engine's own key, read by the watchman. + {State: "a machine's health report", Writer: "the node-engine of the machine, its key `` alone", + KeptIn: "key-value " + delivery.NodeHealthBucket, Others: "read by the watchman through `reads`", + Subjects: kvOf(delivery.NodeHealthBucket), Writes: ownNodeKeys}, // A node's declared packages (novox/hq ADR 0565 clause 6): its engine's own key, read by pacman's verbs. {State: "a machine's declared packages", Writer: "the node-engine of the machine, its key `` alone", KeptIn: "key-value " + EnginePackagesBucket, Others: "read by pacman through `reads`", diff --git a/go/delivery/control_records_test.go b/go/delivery/control_records_test.go index 436867f..1d21efa 100644 --- a/go/delivery/control_records_test.go +++ b/go/delivery/control_records_test.go @@ -3,6 +3,7 @@ package delivery import ( "encoding/json" "testing" + "time" ) // ADR 0565 clauses 2 and 6 (issue 1953 L5-S2): the control records' layouts read what their writers write. The @@ -22,6 +23,14 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) { {"a held value", `{"pull":"novox/mesh-catalog#42","head":"abc","declaration":"sha256:d","node":"novox","module":"mail",` + `"key":"quota","value":20,"by":"person.jochen","via":"telegram","why":"more room","approved":"the operator",` + `"proposal":"p-1","at":"2026-10-29T12:00:00Z"}`, &KeptValue{}}, + // A node's health report as mesh-host 65fbc1e6 encodes it (its test's look, struct order kept). + {"a node's health report", `{"v":1,"node":"anchor","network":{"state":"unhealthy","since":"2026-10-29T20:00:00Z",` + + `"parts":[{"part":"resolv-conf","state":"unhealthy","writer":"NetworkManager","owner":"dnsmasq","since":"2026-10-29T20:00:00Z"}]},` + + `"peers":[{"key":"A=","addresses":["10.10.0.2/32"],"seen":false},{"key":"B=","seen":true}],` + + `"units":{"state":"degraded","failed":[{"unit":"cups.service","scope":"system","since":"2026-10-29T20:00:00Z"}]},` + + `"relogin":[{"account":"jochen","module":"brother","resource":"brother.scanner","reason":"relogin needed: the session began before scanner","since":"2026-10-29T20:00:00Z"}],` + + `"found":[{"module":"photos","resource":"photos.library","path":"/srv/photos","reason":"used as found: owned by root","since":"2026-10-29T20:00:00Z"}],` + + `"at":"2026-10-29T20:00:00Z"}`, &NodeHealth{}}, {"the predefined list", `{"v":1,"commit":"abc","at":"2026-10-29T12:00:00Z","list":{"domain":{"type":"string"}}}`, &PredefinedKept{}}, } { if err := json.Unmarshal([]byte(c.raw), c.into); err != nil { @@ -31,10 +40,17 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) { var want, got any _ = json.Unmarshal([]byte(c.raw), &want) _ = json.Unmarshal(again, &got) - if string(jsonOf(want)) != string(jsonOf(got)) || (c.name == "declared packages" && string(again) != c.raw) { + if string(jsonOf(want)) != string(jsonOf(got)) || ((c.name == "declared packages" || c.name == "a node's health report") && string(again) != c.raw) { t.Errorf("%s reads back as %s", c.name, again) } } + if a, b := (NodeHealth{Node: "anchor", At: time.Now()}), (NodeHealth{Node: "anchor"}); !SameNodeHealth(a, b) || + SameNodeHealth(a, NodeHealth{Node: "anchor", Peers: []PeerSeen{{Key: "A="}}}) { + t.Error("two reports differing only in when they were written are not the same, or differing ones are") + } + if _, err := EncodeNodeHealth(NodeHealth{}); err == nil { + t.Error("a report naming no node was encoded") + } if k, err := HeldValueKey("novox/mesh-catalog#42", "novox", "mail", "quota"); k != "novox.mesh-catalog.42.novox.mail.quota" || err != nil { t.Errorf("a held value's key is %q, %v", k, err) } diff --git a/go/delivery/nodehealth.go b/go/delivery/nodehealth.go new file mode 100644 index 0000000..f41486a --- /dev/null +++ b/go/delivery/nodehealth.go @@ -0,0 +1,146 @@ +package delivery + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "time" +) + +// **A node's own health report on the bus** (issue 774, the operator's alert table of 2026-10-29): one key per node, +// ``, of the node-engine's state `health` (NodeHealthBucket), written by that node's engine alone and only when +// what it says changed. The watchman (seat mesh-watchdogs) reads it for six alerts the controller raised from the +// passing health message: machine-network (Network), network-rewritten (the resolver file's part, Writer and Owner), +// network-unreachable (Peers, read across every node's report), machine-units (Units), relogin-needed (Relogin) and +// directory-used-as-found (Found). Moved here from mesh-host internal/link (65fbc1e6) unchanged; the engine builds a +// report from its own health statement there (NodeHealthOf). + +// NodeHealthState and NodeHealthBucket are the node-engine's state health, as a reader names it and as the bus does. +const ( + NodeHealthState = "health" + NodeHealthBucket = "mesh-host_" + NodeHealthState + NodeHealthRead = "mesh-host." + NodeHealthState +) + +// NodeHealthVersion is the version of NodeHealth's layout. +const NodeHealthVersion = 1 + +// NodeHealth is one node's health report, as its engine last found it. +type NodeHealth struct { + V int `json:"v"` + Node string `json:"node"` + // Network is the machine's networking, its parts without their streaks; nil where the engine does not judge it yet. + Network *NetworkHealth `json:"network,omitempty"` + // Peers are the mesh tunnel's peers on the last look that read them, sorted by key. + Peers []PeerSeen `json:"peers,omitempty"` + // Units is the machine's service managers: each failed unit no module places. + Units *UnitsHealth `json:"units,omitempty"` + // Relogin are the accounts whose running session needs a new login for a group it was given. + Relogin []NodeAccount `json:"relogin,omitempty"` + // Found are the directories used as found. + Found []NodeDirectory `json:"found,omitempty"` + // At is when the engine wrote this, on a change: never a reason to write. + At time.Time `json:"at"` +} + +// NetworkHealth is the machine's networking (ADR 0241): the worst of its parts, since when, and each part. +type NetworkHealth struct { + State string `json:"state"` + Since time.Time `json:"since"` + Parts []NetworkPart `json:"parts"` +} + +// NetworkPart is one part of a machine's networking, as its engine judged it on its second look. +type NetworkPart struct { + // Part is resolv-conf, names, tunnel, bus or route. + Part string `json:"part"` + State string `json:"state"` + // Reason is why it is not healthy, in words with no address, path or domain; Said the detail. + Reason string `json:"reason,omitempty"` + Said string `json:"said,omitempty"` + // Writer is the program that rewrote the resolver file, when it can be named; Owner the module whose file it is. + Writer string `json:"writer,omitempty"` + Owner string `json:"owner,omitempty"` + // Toward is what the failure points at: "hub", or each resolver's address that failed. + Toward []string `json:"toward,omitempty"` + Since time.Time `json:"since"` + Streak int `json:"streak,omitempty"` +} + +// UnitsHealth is the machine's service managers (issue 315). +type UnitsHealth struct { + // State is running, degraded or unknown. + State string `json:"state"` + // Failed is every unit failed on two looks in a row that no module places. + Failed []FailedUnit `json:"failed"` + // Unread names a manager that could not be read, with why; Away an account's own manager not running. + Unread []string `json:"unread,omitempty"` + Away []string `json:"away,omitempty"` +} + +// FailedUnit is one failed unit no module places. +type FailedUnit struct { + Unit string `json:"unit"` + // Scope is system, or user: an account's own manager. + Scope string `json:"scope"` + Load string `json:"load,omitempty"` + Result string `json:"result,omitempty"` + Resource string `json:"resource,omitempty"` + Since time.Time `json:"since"` +} + +// PeerSeen is one peer of the mesh's tunnel: its public key, its allowed addresses, and whether this machine saw a +// handshake with it within five minutes. +type PeerSeen struct { + Key string `json:"key"` + Addresses []string `json:"addresses,omitempty"` + Seen bool `json:"seen"` +} + +// NodeAccount is one account marked relogin needed, with the module whose resource it is and why. +type NodeAccount struct { + Account string `json:"account"` + Module string `json:"module,omitempty"` + Resource string `json:"resource,omitempty"` + Reason string `json:"reason"` + Since time.Time `json:"since"` +} + +// NodeDirectory is one directory used as found: the module declaring it, its resource and path, and why. +type NodeDirectory struct { + Module string `json:"module"` + Resource string `json:"resource"` + Path string `json:"path"` + Reason string `json:"reason"` + Since time.Time `json:"since"` +} + +// EncodeNodeHealth is h as JSON at its version. +func EncodeNodeHealth(h NodeHealth) ([]byte, error) { + h.V = NodeHealthVersion + if h.Node == "" { + return nil, errors.New("a node's health report names its node") + } + return json.Marshal(h) +} + +// DecodeNodeHealth reads a report. +func DecodeNodeHealth(raw []byte) (NodeHealth, error) { + var h NodeHealth + if err := json.Unmarshal(raw, &h); err != nil { + return NodeHealth{}, fmt.Errorf("a node's health report is not readable: %w", err) + } + if h.V != NodeHealthVersion { + return NodeHealth{}, fmt.Errorf("a node's health report of version %d, and this reads %d", h.V, NodeHealthVersion) + } + return h, nil +} + +// SameNodeHealth says two reports say the same, whenever each was written. +func SameNodeHealth(a, b NodeHealth) bool { + a.At, b.At = time.Time{}, time.Time{} + ra, errA := json.Marshal(a) + rb, errB := json.Marshal(b) + return errA == nil && errB == nil && bytes.Equal(ra, rb) +}