Move restic's last good night per item into the sdk's health layouts, so restic's night process writes it and the self-check reads one layout (issue 1955, ADR 0571 clause 3)
This commit is contained in:
@@ -614,7 +614,7 @@ func healthChecks(t *testing.T) {
|
||||
users, err := Users(Records{Nodes: []string{"a"}, Assigned: map[string][]Declared{"a": {
|
||||
{Module: "mesh-conditions", State: []Bucket{bucket("mesh-conditions", "conditions"), bucket("mesh-conditions", "takeover")},
|
||||
Reads: []string{"mesh-self-check.findings", "mesh-healers.attempts", ControllerConditions}},
|
||||
{Module: "mesh-self-check", State: []Bucket{bucket("mesh-self-check", "findings")}},
|
||||
{Module: "mesh-self-check", State: []Bucket{bucket("mesh-self-check", "findings")}, Reads: []string{"restic.good"}},
|
||||
{Module: "mesh-healers", State: []Bucket{bucket("mesh-healers", "findings"), bucket("mesh-healers", "attempts"), bucket("mesh-healers", "would")}},
|
||||
{Module: WatchmanModule, State: []Bucket{bucket(WatchmanModule, "findings")}, Reads: []string{delivery.NodeHealthRead}},
|
||||
{Module: "mesh-hand-acts", State: []Bucket{bucket("mesh-hand-acts", "warrants")}, Logs: []Log{{Module: "mesh-hand-acts", Name: "acts"}}},
|
||||
@@ -630,7 +630,7 @@ func healthChecks(t *testing.T) {
|
||||
{Module: delivery.AssignerModule, State: []Bucket{bucket(delivery.AssignerModule, delivery.SeatHoldersState)}},
|
||||
// ADR 0571 clause 3: disk-load keeps the readings; the backup holder reads them.
|
||||
{Module: "disk-load", State: []Bucket{bucket("disk-load", "readings")}},
|
||||
{Module: "restic", Reads: []string{"disk-load.readings"}},
|
||||
{Module: "restic", Reads: []string{"disk-load.readings"}, Processes: []ProcessPrincipal{{Process: "night", State: []string{"good"}}}},
|
||||
// ADR 0571 clauses 1, 9: the indexer reads the builder's log through its own consumer; the catalogue module asks
|
||||
// the builder's tool.
|
||||
{Module: delivery.BuilderModule, Logs: []Log{{Module: delivery.BuilderModule, Name: delivery.BuilderLog, MaxAgeDays: 30}}},
|
||||
@@ -652,6 +652,9 @@ func healthChecks(t *testing.T) {
|
||||
"$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.$KV.mesh-provisioner_plans.a.keycloak"},
|
||||
"a.mesh-hand-acts": {"$KV.mesh-hand-acts_warrants.mesh-config.ask-1", "mesh.log.mesh-hand-acts.acts.act-1"},
|
||||
"a.disk-load": {"$KV.disk-load_readings.a.pg.data"},
|
||||
// ADR 0571 clause 3: restic's night writes each item's last good night; the self-check reads it.
|
||||
"a.restic.night": {"$KV.restic_good.a.pg.data"},
|
||||
"a.mesh-self-check": {"$JS.API.DIRECT.GET.KV_restic_good.$KV.restic_good.a.pg.data"},
|
||||
"a.restic": {"$JS.API.DIRECT.GET.KV_disk-load_readings.$KV.disk-load_readings.a.pg.data",
|
||||
// ADR 0571 clause 7: its own membership from the grants part's state.
|
||||
"$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.a.restic"},
|
||||
@@ -670,12 +673,16 @@ func healthChecks(t *testing.T) {
|
||||
mayNot := map[string][]string{
|
||||
// The keeper writes no findings and no attempts; no observer writes the keeper's buckets (clause 2).
|
||||
"a.mesh-conditions": {"$KV.mesh-self-check_findings.x", "$KV.mesh-healers_attempts.x", "$KV.mesh-controller_conditions.x"},
|
||||
"a.mesh-self-check": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch"},
|
||||
"a.mesh-self-check": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch", "$KV.restic_good.a.pg.data"},
|
||||
// Only mesh-provisioner writes provisions and plans, and it writes none of the grants part's states (ADR 0570
|
||||
// clauses 1, 8).
|
||||
"a.mesh-provisioner": {"$KV.mesh-grants_accounts.a.m", "$KV.mesh-grants_user-list.sealed"},
|
||||
"a.mesh-vault": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak",
|
||||
"mesh.seat.artifact-store.event.artifacts-gone"},
|
||||
// Each writes its own state alone, and these users are composed.
|
||||
"a.restic.night": {"$KV.disk-load_readings.a.pg.data"},
|
||||
"a.disk-load": {"$KV.restic_good.a.pg.data"},
|
||||
"a.mesh-delivery-builder": {"$KV.disk-load_readings.a.pg.data"},
|
||||
// Only the assigner writes seat-holders.
|
||||
"a.artifact-store-tools": {"$KV.mesh-delivery-assigner_seat-holders.mesh-broker"},
|
||||
// Only disk-load writes its readings.
|
||||
|
||||
@@ -92,6 +92,15 @@ func TestTheHealthLayoutsHoldTheKeepersRules(t *testing.T) {
|
||||
if ReadingKey("ace", "pg", "data") != "ace.pg.data" || ReadingsBucket != "disk-load_readings" {
|
||||
t.Fatal("the readings' key or bucket moved")
|
||||
}
|
||||
// And restic's last good night per item, keyed as a reading.
|
||||
goodRow, _ := json.Marshal(GoodRow{V: GoodVersion, Node: "ace", Module: "pg", Item: "data", CoveredBy: "/srv/pg", LastGood: at})
|
||||
if g, err := DecodeGoodRow(goodRow); err != nil || !g.LastGood.Equal(at) || GoodKey("ace", "pg", "data") != "ace.pg.data" ||
|
||||
GoodBucket != "restic_good" || GoodRead != "restic.good" {
|
||||
t.Fatalf("a last good night read back as %+v: %v", g, err)
|
||||
}
|
||||
if _, err := DecodeGoodRow([]byte(`{"v":2}`)); err == nil {
|
||||
t.Fatal("a last good night of another version was read")
|
||||
}
|
||||
good := Finding{Key: "provider.keycloak.ace.mesh-mail.failing", Kind: "provider-failing", Token: "failing",
|
||||
Subject: Subject{Scope: ScopeProvider, ID: "keycloak.ace.mesh-mail", Machine: "ace"},
|
||||
Severity: Warning, Summary: "keycloak on ace fails mesh-mail", Source: "S8", Observer: WatchmanModule, At: at}
|
||||
|
||||
@@ -125,3 +125,41 @@ func NextReading(prev *Reading, node string, it catalogue.PlacedDataItem, m Meas
|
||||
a.MeasuredAt, b.MeasuredAt = nil, nil
|
||||
return r, !reflect.DeepEqual(a, b)
|
||||
}
|
||||
|
||||
// **Each item's last good night** (novox/hq ADR 0571 clause 3, ADR 0564 §6; the lead, 2026-10-29): the node-backup holder
|
||||
// restic keeps, from its night's process, its own state `good`, one key `<node>.<module>.<item>` for every item a module
|
||||
// declares that a backup path covers, written when it changes: the item, the path whose snapshot keeps it and the newest
|
||||
// good night of its module (zero before the first). The self-check's probe of a stale backup reads it there. Moved here
|
||||
// from mesh-catalog modules/restic (e5ef3bed).
|
||||
const (
|
||||
ResticModule = "restic"
|
||||
GoodState = "good"
|
||||
GoodBucket = ResticModule + "_" + GoodState
|
||||
GoodRead = ResticModule + "." + GoodState
|
||||
GoodVersion = 1
|
||||
)
|
||||
|
||||
// GoodRow is one key of restic's `good`.
|
||||
type GoodRow struct {
|
||||
V int `json:"v"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Item string `json:"item"`
|
||||
CoveredBy string `json:"covered_by"`
|
||||
LastGood time.Time `json:"last_good,omitzero"`
|
||||
}
|
||||
|
||||
// GoodKey is a row's key: `<node>.<module>.<item>`, as a reading's.
|
||||
func GoodKey(node, module, item string) string { return ReadingKey(node, module, item) }
|
||||
|
||||
// DecodeGoodRow reads a row; it refuses another version.
|
||||
func DecodeGoodRow(raw []byte) (GoodRow, error) {
|
||||
var g GoodRow
|
||||
if err := json.Unmarshal(raw, &g); err != nil {
|
||||
return GoodRow{}, fmt.Errorf("health: a last good night is not readable: %w", err)
|
||||
}
|
||||
if g.V != GoodVersion {
|
||||
return GoodRow{}, fmt.Errorf("health: a last good night of version %d, and this reads %d", g.V, GoodVersion)
|
||||
}
|
||||
return g, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user