diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index a83b771..0e37746 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -614,7 +614,7 @@ func healthChecks(t *testing.T) { users, err := Users(Records{Nodes: []string{"a"}, Assigned: map[string][]Declared{"a": { {Module: "mesh-conditions", State: []Bucket{bucket("mesh-conditions", "conditions"), bucket("mesh-conditions", "takeover")}, Reads: []string{"mesh-self-check.findings", "mesh-healers.attempts", ControllerConditions}}, - {Module: "mesh-self-check", State: []Bucket{bucket("mesh-self-check", "findings")}}, + {Module: "mesh-self-check", State: []Bucket{bucket("mesh-self-check", "findings")}, Reads: []string{"restic.good"}}, {Module: "mesh-healers", State: []Bucket{bucket("mesh-healers", "findings"), bucket("mesh-healers", "attempts"), bucket("mesh-healers", "would")}}, {Module: WatchmanModule, State: []Bucket{bucket(WatchmanModule, "findings")}, Reads: []string{delivery.NodeHealthRead}}, {Module: "mesh-hand-acts", State: []Bucket{bucket("mesh-hand-acts", "warrants")}, Logs: []Log{{Module: "mesh-hand-acts", Name: "acts"}}}, @@ -630,7 +630,7 @@ func healthChecks(t *testing.T) { {Module: delivery.AssignerModule, State: []Bucket{bucket(delivery.AssignerModule, delivery.SeatHoldersState)}}, // ADR 0571 clause 3: disk-load keeps the readings; the backup holder reads them. {Module: "disk-load", State: []Bucket{bucket("disk-load", "readings")}}, - {Module: "restic", Reads: []string{"disk-load.readings"}}, + {Module: "restic", Reads: []string{"disk-load.readings"}, Processes: []ProcessPrincipal{{Process: "night", State: []string{"good"}}}}, // ADR 0571 clauses 1, 9: the indexer reads the builder's log through its own consumer; the catalogue module asks // the builder's tool. {Module: delivery.BuilderModule, Logs: []Log{{Module: delivery.BuilderModule, Name: delivery.BuilderLog, MaxAgeDays: 30}}}, @@ -652,6 +652,9 @@ func healthChecks(t *testing.T) { "$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.$KV.mesh-provisioner_plans.a.keycloak"}, "a.mesh-hand-acts": {"$KV.mesh-hand-acts_warrants.mesh-config.ask-1", "mesh.log.mesh-hand-acts.acts.act-1"}, "a.disk-load": {"$KV.disk-load_readings.a.pg.data"}, + // ADR 0571 clause 3: restic's night writes each item's last good night; the self-check reads it. + "a.restic.night": {"$KV.restic_good.a.pg.data"}, + "a.mesh-self-check": {"$JS.API.DIRECT.GET.KV_restic_good.$KV.restic_good.a.pg.data"}, "a.restic": {"$JS.API.DIRECT.GET.KV_disk-load_readings.$KV.disk-load_readings.a.pg.data", // ADR 0571 clause 7: its own membership from the grants part's state. "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.a.restic"}, @@ -670,12 +673,16 @@ func healthChecks(t *testing.T) { mayNot := map[string][]string{ // The keeper writes no findings and no attempts; no observer writes the keeper's buckets (clause 2). "a.mesh-conditions": {"$KV.mesh-self-check_findings.x", "$KV.mesh-healers_attempts.x", "$KV.mesh-controller_conditions.x"}, - "a.mesh-self-check": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch"}, + "a.mesh-self-check": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch", "$KV.restic_good.a.pg.data"}, // Only mesh-provisioner writes provisions and plans, and it writes none of the grants part's states (ADR 0570 // clauses 1, 8). "a.mesh-provisioner": {"$KV.mesh-grants_accounts.a.m", "$KV.mesh-grants_user-list.sealed"}, "a.mesh-vault": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak", "mesh.seat.artifact-store.event.artifacts-gone"}, + // Each writes its own state alone, and these users are composed. + "a.restic.night": {"$KV.disk-load_readings.a.pg.data"}, + "a.disk-load": {"$KV.restic_good.a.pg.data"}, + "a.mesh-delivery-builder": {"$KV.disk-load_readings.a.pg.data"}, // Only the assigner writes seat-holders. "a.artifact-store-tools": {"$KV.mesh-delivery-assigner_seat-holders.mesh-broker"}, // Only disk-load writes its readings. diff --git a/go/health/condition_test.go b/go/health/condition_test.go index 9c25aab..348e62a 100644 --- a/go/health/condition_test.go +++ b/go/health/condition_test.go @@ -92,6 +92,15 @@ func TestTheHealthLayoutsHoldTheKeepersRules(t *testing.T) { if ReadingKey("ace", "pg", "data") != "ace.pg.data" || ReadingsBucket != "disk-load_readings" { t.Fatal("the readings' key or bucket moved") } + // And restic's last good night per item, keyed as a reading. + goodRow, _ := json.Marshal(GoodRow{V: GoodVersion, Node: "ace", Module: "pg", Item: "data", CoveredBy: "/srv/pg", LastGood: at}) + if g, err := DecodeGoodRow(goodRow); err != nil || !g.LastGood.Equal(at) || GoodKey("ace", "pg", "data") != "ace.pg.data" || + GoodBucket != "restic_good" || GoodRead != "restic.good" { + t.Fatalf("a last good night read back as %+v: %v", g, err) + } + if _, err := DecodeGoodRow([]byte(`{"v":2}`)); err == nil { + t.Fatal("a last good night of another version was read") + } good := Finding{Key: "provider.keycloak.ace.mesh-mail.failing", Kind: "provider-failing", Token: "failing", Subject: Subject{Scope: ScopeProvider, ID: "keycloak.ace.mesh-mail", Machine: "ace"}, Severity: Warning, Summary: "keycloak on ace fails mesh-mail", Source: "S8", Observer: WatchmanModule, At: at} diff --git a/go/health/readings.go b/go/health/readings.go index a0983c9..2686259 100644 --- a/go/health/readings.go +++ b/go/health/readings.go @@ -125,3 +125,41 @@ func NextReading(prev *Reading, node string, it catalogue.PlacedDataItem, m Meas a.MeasuredAt, b.MeasuredAt = nil, nil return r, !reflect.DeepEqual(a, b) } + +// **Each item's last good night** (novox/hq ADR 0571 clause 3, ADR 0564 ยง6; the lead, 2026-10-29): the node-backup holder +// restic keeps, from its night's process, its own state `good`, one key `..` for every item a module +// declares that a backup path covers, written when it changes: the item, the path whose snapshot keeps it and the newest +// good night of its module (zero before the first). The self-check's probe of a stale backup reads it there. Moved here +// from mesh-catalog modules/restic (e5ef3bed). +const ( + ResticModule = "restic" + GoodState = "good" + GoodBucket = ResticModule + "_" + GoodState + GoodRead = ResticModule + "." + GoodState + GoodVersion = 1 +) + +// GoodRow is one key of restic's `good`. +type GoodRow struct { + V int `json:"v"` + Node string `json:"node"` + Module string `json:"module"` + Item string `json:"item"` + CoveredBy string `json:"covered_by"` + LastGood time.Time `json:"last_good,omitzero"` +} + +// GoodKey is a row's key: `..`, as a reading's. +func GoodKey(node, module, item string) string { return ReadingKey(node, module, item) } + +// DecodeGoodRow reads a row; it refuses another version. +func DecodeGoodRow(raw []byte) (GoodRow, error) { + var g GoodRow + if err := json.Unmarshal(raw, &g); err != nil { + return GoodRow{}, fmt.Errorf("health: a last good night is not readable: %w", err) + } + if g.V != GoodVersion { + return GoodRow{}, fmt.Errorf("health: a last good night of version %d, and this reads %d", g.V, GoodVersion) + } + return g, nil +}