novox/mesh-sdk
18
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Move the node's health report layout into the sdk and let each node write only its own key while the watchman reads them, so the six machine alerts leave the controller (issue 774)

Browse Source
This commit is contained in:
jschoubben
2026-10-29 22:58:05 +01:00
parent 58a83249c8
commit 94d0de5c19
5 changed files with 184 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go/buscompose/buscompose_test.go
+11 -1
View File
@@ -79,6 +79,8 @@ func engineBeyond(user, g string) bool {
strings.Contains(g, "KV_"+EngineActsBucket) || strings.HasPrefix(g, "$KV."+EngineActsBucket+"."+node+".") ||
// And its node's declared packages (novox/hq ADR 0565 clause 6).
strings.Contains(g, "KV_"+EnginePackagesBucket) || g == "$KV."+EnginePackagesBucket+"."+node ||
// And its own health report (issue 774).
strings.Contains(g, "KV_"+delivery.NodeHealthBucket) || g == "$KV."+delivery.NodeHealthBucket+"."+node ||
// And the provisions of its node's modules, read (novox/hq ADR 0570 clause 6).
strings.Contains(g, "KV_"+provisions.ProvisionsBucket)
}
@@ -601,7 +603,7 @@ func healthChecks(t *testing.T) {
Reads: []string{"mesh-self-check.findings", "mesh-healers.attempts", ControllerConditions}},
{Module: "mesh-self-check", State: []Bucket{bucket("mesh-self-check", "findings")}},
{Module: "mesh-healers", State: []Bucket{bucket("mesh-healers", "findings"), bucket("mesh-healers", "attempts"), bucket("mesh-healers", "would")}},
{Module: WatchmanModule, State: []Bucket{bucket(WatchmanModule, "findings")}},
{Module: WatchmanModule, State: []Bucket{bucket(WatchmanModule, "findings")}, Reads: []string{delivery.NodeHealthRead}},
{Module: "mesh-hand-acts", State: []Bucket{bucket("mesh-hand-acts", "warrants")}, Logs: []Log{{Module: "mesh-hand-acts", Name: "acts"}}},
// ADR 0570 clauses 1, 8 (issue 1954 L6-P5): mesh-provisioner keeps provisions and plans; the vault and a provider
// read them.
@@ -675,6 +677,14 @@ func healthChecks(t *testing.T) {
MayPublish(perms, "$KV.mesh-provisioner_provisions.a.keycloak.db")) {
t.Errorf("node %s reads the provisions %v", u.Node, perms.Publish)
}
// Issue 774: a node's engine writes its own health report's key and no other node's.
if u.Kind == KindNode && (!MayPublish(perms, "$KV.mesh-host_health.a") || MayPublish(perms, "$KV.mesh-host_health.b")) {
t.Errorf("node %s writes the health reports %v", u.Node, perms.Publish)
}
if u.Kind == KindModule && u.Module == WatchmanModule && (!MayPublish(perms, "$JS.API.DIRECT.GET.KV_mesh-host_health.$KV.mesh-host_health.b") ||
MayPublish(perms, "$KV.mesh-host_health.b")) {
t.Errorf("the watchman reads the health reports %v, %v", MayPublish(perms, "$JS.API.DIRECT.GET.KV_mesh-host_health.$KV.mesh-host_health.b"), perms.Publish)
}
if u.Kind == KindNode && (!MayPublish(perms, "$KV.mesh-host_packages.a") || MayPublish(perms, "$KV.mesh-host_packages.b")) {
t.Errorf("node %s writes the packages keys %v", u.Node, perms.Publish)
}
go/buscompose/engine.go
+6
View File
@@ -58,6 +58,12 @@ func engineGrants(p Principal) (pub, sub []string) {
if safeSubject.MatchString(p.Node) {
pub = append(pub, readGrants(provisions.ProvisionsBucket, p.Node+".>")...)
}
// And its node's own health report, its key `<node>` alone (issue 774, the operator's alert table): what the watchman
// reads for the machine's alerts.
if safeSubject.MatchString(p.Node) {
pub = append(pub, append([]string{"$KV." + delivery.NodeHealthBucket + "." + p.Node},
readGrants(delivery.NodeHealthBucket, p.Node)...)...)
}
// And its node's declared packages, its own key alone (ADR 0565 clause 6).
if safeSubject.MatchString(p.Node) {
pub = append(pub, EnginePackagesGrants(p.Node)...)
go/buscompose/writers.go
+4
View File
@@ -248,6 +248,10 @@ var WritersTable = []WriterRow{
Others: "read through `reads` as node-engine.have", Subjects: kvOf(delivery.HaveBucket), Writes: ownNodeKeys},
// The acts a node's engine was asked through its node seat, and its own asks (novox/hq ADR 0563 §2, §3): the
// node-engine's declared state, created by the grants part, written by each node's engine under its own keys.
// A node's own health report (issue 774): its engine's own key, read by the watchman.
{State: "a machine's health report", Writer: "the node-engine of the machine, its key `<node>` alone",
KeptIn: "key-value " + delivery.NodeHealthBucket, Others: "read by the watchman through `reads`",
Subjects: kvOf(delivery.NodeHealthBucket), Writes: ownNodeKeys},
// A node's declared packages (novox/hq ADR 0565 clause 6): its engine's own key, read by pacman's verbs.
{State: "a machine's declared packages", Writer: "the node-engine of the machine, its key `<node>` alone",
KeptIn: "key-value " + EnginePackagesBucket, Others: "read by pacman through `reads`",
go/delivery/control_records_test.go
+17 -1
View File
@@ -3,6 +3,7 @@ package delivery
import (
"encoding/json"
"testing"
"time"
)
// ADR 0565 clauses 2 and 6 (issue 1953 L5-S2): the control records' layouts read what their writers write. The
@@ -22,6 +23,14 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) {
{"a held value", `{"pull":"novox/mesh-catalog#42","head":"abc","declaration":"sha256:d","node":"novox","module":"mail",` +
`"key":"quota","value":20,"by":"person.jochen","via":"telegram","why":"more room","approved":"the operator",` +
`"proposal":"p-1","at":"2026-10-29T12:00:00Z"}`, &KeptValue{}},
// A node's health report as mesh-host 65fbc1e6 encodes it (its test's look, struct order kept).
{"a node's health report", `{"v":1,"node":"anchor","network":{"state":"unhealthy","since":"2026-10-29T20:00:00Z",` +
`"parts":[{"part":"resolv-conf","state":"unhealthy","writer":"NetworkManager","owner":"dnsmasq","since":"2026-10-29T20:00:00Z"}]},` +
`"peers":[{"key":"A=","addresses":["10.10.0.2/32"],"seen":false},{"key":"B=","seen":true}],` +
`"units":{"state":"degraded","failed":[{"unit":"cups.service","scope":"system","since":"2026-10-29T20:00:00Z"}]},` +
`"relogin":[{"account":"jochen","module":"brother","resource":"brother.scanner","reason":"relogin needed: the session began before scanner","since":"2026-10-29T20:00:00Z"}],` +
`"found":[{"module":"photos","resource":"photos.library","path":"/srv/photos","reason":"used as found: owned by root","since":"2026-10-29T20:00:00Z"}],` +
`"at":"2026-10-29T20:00:00Z"}`, &NodeHealth{}},
{"the predefined list", `{"v":1,"commit":"abc","at":"2026-10-29T12:00:00Z","list":{"domain":{"type":"string"}}}`, &PredefinedKept{}},
} {
if err := json.Unmarshal([]byte(c.raw), c.into); err != nil {
@@ -31,10 +40,17 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) {
var want, got any
_ = json.Unmarshal([]byte(c.raw), &want)
_ = json.Unmarshal(again, &got)
if string(jsonOf(want)) != string(jsonOf(got)) || (c.name == "declared packages" && string(again) != c.raw) {
if string(jsonOf(want)) != string(jsonOf(got)) || ((c.name == "declared packages" || c.name == "a node's health report") && string(again) != c.raw) {
t.Errorf("%s reads back as %s", c.name, again)
}
}
if a, b := (NodeHealth{Node: "anchor", At: time.Now()}), (NodeHealth{Node: "anchor"}); !SameNodeHealth(a, b) ||
SameNodeHealth(a, NodeHealth{Node: "anchor", Peers: []PeerSeen{{Key: "A="}}}) {
t.Error("two reports differing only in when they were written are not the same, or differing ones are")
}
if _, err := EncodeNodeHealth(NodeHealth{}); err == nil {
t.Error("a report naming no node was encoded")
}
if k, err := HeldValueKey("novox/mesh-catalog#42", "novox", "mail", "quota"); k != "novox.mesh-catalog.42.novox.mail.quota" || err != nil {
t.Errorf("a held value's key is %q, %v", k, err)
}
go/delivery/nodehealth.go
+146
View File
@@ -0,0 +1,146 @@
package delivery
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"time"
)
// **A node's own health report on the bus** (issue 774, the operator's alert table of 2026-10-29): one key per node,
// `<node>`, of the node-engine's state `health` (NodeHealthBucket), written by that node's engine alone and only when
// what it says changed. The watchman (seat mesh-watchdogs) reads it for six alerts the controller raised from the
// passing health message: machine-network (Network), network-rewritten (the resolver file's part, Writer and Owner),
// network-unreachable (Peers, read across every node's report), machine-units (Units), relogin-needed (Relogin) and
// directory-used-as-found (Found). Moved here from mesh-host internal/link (65fbc1e6) unchanged; the engine builds a
// report from its own health statement there (NodeHealthOf).
// NodeHealthState and NodeHealthBucket are the node-engine's state health, as a reader names it and as the bus does.
const (
NodeHealthState = "health"
NodeHealthBucket = "mesh-host_" + NodeHealthState
NodeHealthRead = "mesh-host." + NodeHealthState
)
// NodeHealthVersion is the version of NodeHealth's layout.
const NodeHealthVersion = 1
// NodeHealth is one node's health report, as its engine last found it.
type NodeHealth struct {
V int `json:"v"`
Node string `json:"node"`
// Network is the machine's networking, its parts without their streaks; nil where the engine does not judge it yet.
Network *NetworkHealth `json:"network,omitempty"`
// Peers are the mesh tunnel's peers on the last look that read them, sorted by key.
Peers []PeerSeen `json:"peers,omitempty"`
// Units is the machine's service managers: each failed unit no module places.
Units *UnitsHealth `json:"units,omitempty"`
// Relogin are the accounts whose running session needs a new login for a group it was given.
Relogin []NodeAccount `json:"relogin,omitempty"`
// Found are the directories used as found.
Found []NodeDirectory `json:"found,omitempty"`
// At is when the engine wrote this, on a change: never a reason to write.
At time.Time `json:"at"`
}
// NetworkHealth is the machine's networking (ADR 0241): the worst of its parts, since when, and each part.
type NetworkHealth struct {
State string `json:"state"`
Since time.Time `json:"since"`
Parts []NetworkPart `json:"parts"`
}
// NetworkPart is one part of a machine's networking, as its engine judged it on its second look.
type NetworkPart struct {
// Part is resolv-conf, names, tunnel, bus or route.
Part string `json:"part"`
State string `json:"state"`
// Reason is why it is not healthy, in words with no address, path or domain; Said the detail.
Reason string `json:"reason,omitempty"`
Said string `json:"said,omitempty"`
// Writer is the program that rewrote the resolver file, when it can be named; Owner the module whose file it is.
Writer string `json:"writer,omitempty"`
Owner string `json:"owner,omitempty"`
// Toward is what the failure points at: "hub", or each resolver's address that failed.
Toward []string `json:"toward,omitempty"`
Since time.Time `json:"since"`
Streak int `json:"streak,omitempty"`
}
// UnitsHealth is the machine's service managers (issue 315).
type UnitsHealth struct {
// State is running, degraded or unknown.
State string `json:"state"`
// Failed is every unit failed on two looks in a row that no module places.
Failed []FailedUnit `json:"failed"`
// Unread names a manager that could not be read, with why; Away an account's own manager not running.
Unread []string `json:"unread,omitempty"`
Away []string `json:"away,omitempty"`
}
// FailedUnit is one failed unit no module places.
type FailedUnit struct {
Unit string `json:"unit"`
// Scope is system, or user: an account's own manager.
Scope string `json:"scope"`
Load string `json:"load,omitempty"`
Result string `json:"result,omitempty"`
Resource string `json:"resource,omitempty"`
Since time.Time `json:"since"`
}
// PeerSeen is one peer of the mesh's tunnel: its public key, its allowed addresses, and whether this machine saw a
// handshake with it within five minutes.
type PeerSeen struct {
Key string `json:"key"`
Addresses []string `json:"addresses,omitempty"`
Seen bool `json:"seen"`
}
// NodeAccount is one account marked relogin needed, with the module whose resource it is and why.
type NodeAccount struct {
Account string `json:"account"`
Module string `json:"module,omitempty"`
Resource string `json:"resource,omitempty"`
Reason string `json:"reason"`
Since time.Time `json:"since"`
}
// NodeDirectory is one directory used as found: the module declaring it, its resource and path, and why.
type NodeDirectory struct {
Module string `json:"module"`
Resource string `json:"resource"`
Path string `json:"path"`
Reason string `json:"reason"`
Since time.Time `json:"since"`
}
// EncodeNodeHealth is h as JSON at its version.
func EncodeNodeHealth(h NodeHealth) ([]byte, error) {
h.V = NodeHealthVersion
if h.Node == "" {
return nil, errors.New("a node's health report names its node")
}
return json.Marshal(h)
}
// DecodeNodeHealth reads a report.
func DecodeNodeHealth(raw []byte) (NodeHealth, error) {
var h NodeHealth
if err := json.Unmarshal(raw, &h); err != nil {
return NodeHealth{}, fmt.Errorf("a node's health report is not readable: %w", err)
}
if h.V != NodeHealthVersion {
return NodeHealth{}, fmt.Errorf("a node's health report of version %d, and this reads %d", h.V, NodeHealthVersion)
}
return h, nil
}
// SameNodeHealth says two reports say the same, whenever each was written.
func SameNodeHealth(a, b NodeHealth) bool {
a.At, b.At = time.Time{}, time.Time{}
ra, errA := json.Marshal(a)
rb, errB := json.Marshal(b)
return errA == nil && errB == nil && bytes.Equal(ra, rb)
}