From 6e5fd647c05766f7813435ee4a470bb832549a00 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:58:11 +0100 Subject: [PATCH] mesh-self-check: D13 raises empty-replacement and cleanup-waiting for retired readings and counts a missing backup from the first measurement; sdk 31e90aa (issue 1952, ADR 0233) --- .../cmd/mesh-self-check/probes_data.go | 54 ++++++++++++++++++- .../cmd/mesh-self-check/probes_test.go | 27 ++++++++++ modules/mesh-self-check/go.mod | 2 +- modules/mesh-self-check/go.sum | 4 +- 4 files changed, 82 insertions(+), 5 deletions(-) diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_data.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_data.go index 23f74c52..98d8146f 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes_data.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_data.go @@ -42,6 +42,7 @@ const ( kindProtectionMissing = "protection-missing" kindCleanupWaiting = "cleanup-waiting" shrinkWindowDays = 7 + cleanupAfter = 30 * 24 * time.Hour shrinkFloor int64 = 16 << 20 ) @@ -194,9 +195,12 @@ func sizeWords(size *int64) string { return "" } -// firstSeen is when an item was first read: the oldest day its seven-day peak is made of, else its measurement. A lower -// bound on how long it has been declared (the controller kept the day it first saw it). +// firstSeen is when an item was first measured on its node (the reading keeps it); for a reading written before it did, +// the oldest day its seven-day peak is made of, else its measurement: a lower bound on how long it has been declared. func firstSeen(r health.Reading, now time.Time) time.Time { + if !r.FirstSeen.IsZero() { + return r.FirstSeen + } if len(r.Days) > 0 { if t, err := time.Parse("2006-01-02", r.Days[0].Day); err == nil { return t @@ -223,6 +227,8 @@ func dataFindings(f dataFacts) []Observation { items []string } shrunkDatasets := map[string]shrunk{} + byItem := map[string][]health.Reading{} + retiredCopies := map[string][]health.Reading{} for _, m := range f.unmeasured { o := obs(kindDataUnmeasured, m, m, Warning, fmt.Sprintf("%s's backup holder did not say what it measured of the data "+ "declared there, so nothing about that data is known", m), "disk-load keeps no reading of this machine's data") @@ -237,6 +243,19 @@ func dataFindings(f dataFacts) []Observation { class := r.Class item, declared := f.manifests[r.Module].DataItem(r.ID) id := keyOfItem(r.Node, r.Module, r.ID) + if !r.RetiredAt.IsZero() { + retiredCopies[r.Module+"/"+r.ID] = append(retiredCopies[r.Module+"/"+r.ID], r) + if f.now.Sub(r.RetiredAt) > cleanupAfter && catalogue.Watched(class) { + out = append(out, obs(kindCleanupWaiting, id, r.Node, Warning, + fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept where it was since %s; "+ + "`cleanup delete %s %s %s --why …` once a person has decided, or assign %s there again", + r.ID, r.Module, r.Node, r.Class, sizeWords(r.SizeBytes), int(f.now.Sub(r.RetiredAt).Hours()/24), + r.RetiredWhy, r.Node, r.Module, r.ID, r.Module), "kept at "+orUnknownPath(r.Path))) + out[len(out)-1].Token = "cleanup" + } + continue + } + byItem[r.Module+"/"+r.ID] = append(byItem[r.Module+"/"+r.ID], r) if !catalogue.Watched(class) { continue } @@ -338,5 +357,36 @@ func dataFindings(f dataFacts) []Observation { machine, severityOf(class), fmt.Sprintf("the %s storage on %s that protects %s %s", red.Kind, machine, strings.Join(names, ", "), state), fmt.Sprintf("the %s storage %s %s: %s", red.Kind, red.Where, state, firstLine(red.Said)))) } + // The same item on several machines: a copy that is in use and far smaller than one kept retired elsewhere is an + // empty replacement. Only against a retired copy — a module running on two machines on purpose keeps two different + // sets of data. + items := make([]string, 0, len(byItem)) + for k := range byItem { + items = append(items, k) + } + sort.Strings(items) + for _, key := range items { + for _, a := range byItem[key] { + if a.SizeBytes == nil || !catalogue.Watched(a.Class) || !comparable(a.Precision) { + continue + } + for _, o := range retiredCopies[key] { + if o.Node == a.Node || o.SizeBytes == nil || !comparable(o.Precision) || !replacedByLess(*a.SizeBytes, *o.SizeBytes) { + continue + } + ob := obs(kindEmptyReplacement, keyOfItem(a.Node, a.Module, a.ID), a.Node, severityOf(a.Class), + fmt.Sprintf("%s of %s on %s holds %s, and the copy %s kept on %s holds %s: %s is running on "+ + "an empty replacement of its data. Move the data, or assign it back where its data is", + a.ID, a.Module, a.Node, sizeWords(a.SizeBytes), o.Module, o.Node, sizeWords(o.SizeBytes), a.Module), "") + ob.Subject.Also = []string{o.Node} + out = append(out, ob) + break + } + } + } return out } + +// replacedByLess is whether a copy in use is an empty replacement of a copy kept elsewhere: less than half of it, and at +// least shrinkFloor less. +func replacedByLess(inUse, kept int64) bool { return inUse*2 < kept && kept-inUse >= shrinkFloor } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go index 7c3a9de8..03eb637a 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go @@ -187,6 +187,22 @@ func TestTheBusProbes(t *testing.T) { r.size("ace", "plex", "thumbs", 10<<20, 600<<20).dataset("ace", "plex", "thumbs", "storage/media") r.size("ace", "plex", "meta", 10<<20, 600<<20).dataset("ace", "plex", "meta", "storage/media") })}, + {"D13 a retired item is kept, not judged, and cleanup waits after thirty days", probeData, bus{}, []string{ + "machine.ace.pg.dumps.cleanup"}, "", dataMesh(func(r *dataRig) { + r.size("ace", "pg", "dumps", 1<<20, 1<<30).retired("ace", "pg", "dumps", -31*24*time.Hour) + r.size("ace", "pg", "old", 1<<20, 1<<30).retired("ace", "pg", "old", -2*24*time.Hour) + })}, + {"D13 a copy far smaller than a retired one elsewhere is an empty replacement", probeData, bus{}, + []string{"machine.ace.plex.thumbs.empty-replacement"}, "", dataMesh(func(r *dataRig) { + r.size("ace", "plex", "thumbs", 10<<20, 10<<20) + r.size("g14", "plex", "thumbs", 600<<20, 600<<20).retired("g14", "plex", "thumbs", -24*time.Hour) + })}, + {"D13 no good backup is counted from when the item was first measured", probeData, bus{}, + []string{"machine.ace.pg.data.backup-stale"}, "", dataMesh(func(r *dataRig) { + delete(r.good, health.ReadingKey("ace", "pg", "data")) + delete(r.good, health.ReadingKey("ace", "plex", "meta")) + r.firstSeen("ace", "pg", "data", -100*time.Hour).firstSeen("ace", "plex", "meta", -time.Hour) + })}, {"D13 a machine with a backup holder and no reading", probeData, bus{}, []string{"machine.g14.data-unmeasured"}, "", dataMesh(func(r *dataRig) { r.holder("g14") })}, // D-fill (ADR 0297 §6): a module's bucket or log at three quarters of its cap, kept until under seven tenths. @@ -375,6 +391,17 @@ func (r *dataRig) lastGood(node, module, item string, ago time.Duration) *dataRi return r } +func (r *dataRig) retired(node, module, item string, ago time.Duration) *dataRig { + x := r.reading(node, module, item) + x.RetiredAt, x.RetiredWhy = dataNow.Add(ago), "no longer declared on "+node + return r +} + +func (r *dataRig) firstSeen(node, module, item string, ago time.Duration) *dataRig { + r.reading(node, module, item).FirstSeen = dataNow.Add(ago) + return r +} + func (r *dataRig) holder(node string) { key, _ := delivery.SeatHoldersKey("node-backup", node) _ = r.s.Put(SeatHoldersRead, key, delivery.SeatHolders{V: 1, Seat: "node-backup", Scope: "node", Node: node, diff --git a/modules/mesh-self-check/go.mod b/modules/mesh-self-check/go.mod index 6d01076d..e2fbe713 100644 --- a/modules/mesh-self-check/go.mod +++ b/modules/mesh-self-check/go.mod @@ -3,7 +3,7 @@ module mesh-self-check go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029224808-f3677eb6f36e + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029225606-31e90aaa0f0d github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-self-check/go.sum b/modules/mesh-self-check/go.sum index 139d8eaa..bb9fb254 100644 --- a/modules/mesh-self-check/go.sum +++ b/modules/mesh-self-check/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029224808-f3677eb6f36e h1:h8bh7ZyAgPuFC9/lc6iFBD/hduzomLodZ0a1oJZ5X4U= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029224808-f3677eb6f36e/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029225606-31e90aaa0f0d h1:HHSXxRS+GEYNgtdg9MFB8W2Y6ycPJF7YLF46y3oj7Ow= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029225606-31e90aaa0f0d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=