Move restic's last good night per item into the sdk's health layouts, so restic's night process writes it and the self-check reads one layout (issue 1955, ADR 0571 clause 3)

This commit is contained in:
2026-10-29 23:37:37 +01:00
parent 08fc70783a
commit 9f0bb1cfa4
3 changed files with 57 additions and 3 deletions
+10 -3
View File
@@ -614,7 +614,7 @@ func healthChecks(t *testing.T) {
users, err := Users(Records{Nodes: []string{"a"}, Assigned: map[string][]Declared{"a": {
{Module: "mesh-conditions", State: []Bucket{bucket("mesh-conditions", "conditions"), bucket("mesh-conditions", "takeover")},
Reads: []string{"mesh-self-check.findings", "mesh-healers.attempts", ControllerConditions}},
{Module: "mesh-self-check", State: []Bucket{bucket("mesh-self-check", "findings")}},
{Module: "mesh-self-check", State: []Bucket{bucket("mesh-self-check", "findings")}, Reads: []string{"restic.good"}},
{Module: "mesh-healers", State: []Bucket{bucket("mesh-healers", "findings"), bucket("mesh-healers", "attempts"), bucket("mesh-healers", "would")}},
{Module: WatchmanModule, State: []Bucket{bucket(WatchmanModule, "findings")}, Reads: []string{delivery.NodeHealthRead}},
{Module: "mesh-hand-acts", State: []Bucket{bucket("mesh-hand-acts", "warrants")}, Logs: []Log{{Module: "mesh-hand-acts", Name: "acts"}}},
@@ -630,7 +630,7 @@ func healthChecks(t *testing.T) {
{Module: delivery.AssignerModule, State: []Bucket{bucket(delivery.AssignerModule, delivery.SeatHoldersState)}},
// ADR 0571 clause 3: disk-load keeps the readings; the backup holder reads them.
{Module: "disk-load", State: []Bucket{bucket("disk-load", "readings")}},
{Module: "restic", Reads: []string{"disk-load.readings"}},
{Module: "restic", Reads: []string{"disk-load.readings"}, Processes: []ProcessPrincipal{{Process: "night", State: []string{"good"}}}},
// ADR 0571 clauses 1, 9: the indexer reads the builder's log through its own consumer; the catalogue module asks
// the builder's tool.
{Module: delivery.BuilderModule, Logs: []Log{{Module: delivery.BuilderModule, Name: delivery.BuilderLog, MaxAgeDays: 30}}},
@@ -652,6 +652,9 @@ func healthChecks(t *testing.T) {
"$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.$KV.mesh-provisioner_plans.a.keycloak"},
"a.mesh-hand-acts": {"$KV.mesh-hand-acts_warrants.mesh-config.ask-1", "mesh.log.mesh-hand-acts.acts.act-1"},
"a.disk-load": {"$KV.disk-load_readings.a.pg.data"},
// ADR 0571 clause 3: restic's night writes each item's last good night; the self-check reads it.
"a.restic.night": {"$KV.restic_good.a.pg.data"},
"a.mesh-self-check": {"$JS.API.DIRECT.GET.KV_restic_good.$KV.restic_good.a.pg.data"},
"a.restic": {"$JS.API.DIRECT.GET.KV_disk-load_readings.$KV.disk-load_readings.a.pg.data",
// ADR 0571 clause 7: its own membership from the grants part's state.
"$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.a.restic"},
@@ -670,12 +673,16 @@ func healthChecks(t *testing.T) {
mayNot := map[string][]string{
// The keeper writes no findings and no attempts; no observer writes the keeper's buckets (clause 2).
"a.mesh-conditions": {"$KV.mesh-self-check_findings.x", "$KV.mesh-healers_attempts.x", "$KV.mesh-controller_conditions.x"},
"a.mesh-self-check": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch"},
"a.mesh-self-check": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch", "$KV.restic_good.a.pg.data"},
// Only mesh-provisioner writes provisions and plans, and it writes none of the grants part's states (ADR 0570
// clauses 1, 8).
"a.mesh-provisioner": {"$KV.mesh-grants_accounts.a.m", "$KV.mesh-grants_user-list.sealed"},
"a.mesh-vault": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak",
"mesh.seat.artifact-store.event.artifacts-gone"},
// Each writes its own state alone, and these users are composed.
"a.restic.night": {"$KV.disk-load_readings.a.pg.data"},
"a.disk-load": {"$KV.restic_good.a.pg.data"},
"a.mesh-delivery-builder": {"$KV.disk-load_readings.a.pg.data"},
// Only the assigner writes seat-holders.
"a.artifact-store-tools": {"$KV.mesh-delivery-assigner_seat-holders.mesh-broker"},
// Only disk-load writes its readings.
+9
View File
@@ -92,6 +92,15 @@ func TestTheHealthLayoutsHoldTheKeepersRules(t *testing.T) {
if ReadingKey("ace", "pg", "data") != "ace.pg.data" || ReadingsBucket != "disk-load_readings" {
t.Fatal("the readings' key or bucket moved")
}
// And restic's last good night per item, keyed as a reading.
goodRow, _ := json.Marshal(GoodRow{V: GoodVersion, Node: "ace", Module: "pg", Item: "data", CoveredBy: "/srv/pg", LastGood: at})
if g, err := DecodeGoodRow(goodRow); err != nil || !g.LastGood.Equal(at) || GoodKey("ace", "pg", "data") != "ace.pg.data" ||
GoodBucket != "restic_good" || GoodRead != "restic.good" {
t.Fatalf("a last good night read back as %+v: %v", g, err)
}
if _, err := DecodeGoodRow([]byte(`{"v":2}`)); err == nil {
t.Fatal("a last good night of another version was read")
}
good := Finding{Key: "provider.keycloak.ace.mesh-mail.failing", Kind: "provider-failing", Token: "failing",
Subject: Subject{Scope: ScopeProvider, ID: "keycloak.ace.mesh-mail", Machine: "ace"},
Severity: Warning, Summary: "keycloak on ace fails mesh-mail", Source: "S8", Observer: WatchmanModule, At: at}
+38
View File
@@ -125,3 +125,41 @@ func NextReading(prev *Reading, node string, it catalogue.PlacedDataItem, m Meas
a.MeasuredAt, b.MeasuredAt = nil, nil
return r, !reflect.DeepEqual(a, b)
}
// **Each item's last good night** (novox/hq ADR 0571 clause 3, ADR 0564 §6; the lead, 2026-10-29): the node-backup holder
// restic keeps, from its night's process, its own state `good`, one key `<node>.<module>.<item>` for every item a module
// declares that a backup path covers, written when it changes: the item, the path whose snapshot keeps it and the newest
// good night of its module (zero before the first). The self-check's probe of a stale backup reads it there. Moved here
// from mesh-catalog modules/restic (e5ef3bed).
const (
ResticModule = "restic"
GoodState = "good"
GoodBucket = ResticModule + "_" + GoodState
GoodRead = ResticModule + "." + GoodState
GoodVersion = 1
)
// GoodRow is one key of restic's `good`.
type GoodRow struct {
V int `json:"v"`
Node string `json:"node"`
Module string `json:"module"`
Item string `json:"item"`
CoveredBy string `json:"covered_by"`
LastGood time.Time `json:"last_good,omitzero"`
}
// GoodKey is a row's key: `<node>.<module>.<item>`, as a reading's.
func GoodKey(node, module, item string) string { return ReadingKey(node, module, item) }
// DecodeGoodRow reads a row; it refuses another version.
func DecodeGoodRow(raw []byte) (GoodRow, error) {
var g GoodRow
if err := json.Unmarshal(raw, &g); err != nil {
return GoodRow{}, fmt.Errorf("health: a last good night is not readable: %w", err)
}
if g.V != GoodVersion {
return GoodRow{}, fmt.Errorf("health: a last good night of version %d, and this reads %d", g.V, GoodVersion)
}
return g, nil
}