novox/mesh-catalog
18
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

mesh-self-check: D13 raises empty-replacement and cleanup-waiting for retired readings and counts a missing backup from the first measurement; sdk 31e90aa (issue 1952, ADR 0233)

Browse Source
This commit is contained in:
jschoubben
2026-10-29 23:58:11 +01:00
parent 936e7308bf
commit 6e5fd647c0
4 changed files with 82 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-self-check/cmd/mesh-self-check/probes_data.go
+52 -2
View File
@@ -42,6 +42,7 @@ const (
kindProtectionMissing = "protection-missing"
kindCleanupWaiting = "cleanup-waiting"
shrinkWindowDays = 7
cleanupAfter = 30 * 24 * time.Hour
shrinkFloor int64 = 16 << 20
)
@@ -194,9 +195,12 @@ func sizeWords(size *int64) string {
return ""
}
// firstSeen is when an item was first read: the oldest day its seven-day peak is made of, else its measurement. A lower
// bound on how long it has been declared (the controller kept the day it first saw it).
// firstSeen is when an item was first measured on its node (the reading keeps it); for a reading written before it did,
// the oldest day its seven-day peak is made of, else its measurement: a lower bound on how long it has been declared.
func firstSeen(r health.Reading, now time.Time) time.Time {
if !r.FirstSeen.IsZero() {
return r.FirstSeen
}
if len(r.Days) > 0 {
if t, err := time.Parse("2006-01-02", r.Days[0].Day); err == nil {
return t
@@ -223,6 +227,8 @@ func dataFindings(f dataFacts) []Observation {
items []string
}
shrunkDatasets := map[string]shrunk{}
byItem := map[string][]health.Reading{}
retiredCopies := map[string][]health.Reading{}
for _, m := range f.unmeasured {
o := obs(kindDataUnmeasured, m, m, Warning, fmt.Sprintf("%s's backup holder did not say what it measured of the data "+
"declared there, so nothing about that data is known", m), "disk-load keeps no reading of this machine's data")
@@ -237,6 +243,19 @@ func dataFindings(f dataFacts) []Observation {
class := r.Class
item, declared := f.manifests[r.Module].DataItem(r.ID)
id := keyOfItem(r.Node, r.Module, r.ID)
if !r.RetiredAt.IsZero() {
retiredCopies[r.Module+"/"+r.ID] = append(retiredCopies[r.Module+"/"+r.ID], r)
if f.now.Sub(r.RetiredAt) > cleanupAfter && catalogue.Watched(class) {
out = append(out, obs(kindCleanupWaiting, id, r.Node, Warning,
fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept where it was since %s; "+
"`cleanup delete %s %s %s --why …` once a person has decided, or assign %s there again",
r.ID, r.Module, r.Node, r.Class, sizeWords(r.SizeBytes), int(f.now.Sub(r.RetiredAt).Hours()/24),
r.RetiredWhy, r.Node, r.Module, r.ID, r.Module), "kept at "+orUnknownPath(r.Path)))
out[len(out)-1].Token = "cleanup"
}
continue
}
byItem[r.Module+"/"+r.ID] = append(byItem[r.Module+"/"+r.ID], r)
if !catalogue.Watched(class) {
continue
}
@@ -338,5 +357,36 @@ func dataFindings(f dataFacts) []Observation {
machine, severityOf(class), fmt.Sprintf("the %s storage on %s that protects %s %s", red.Kind, machine,
strings.Join(names, ", "), state), fmt.Sprintf("the %s storage %s %s: %s", red.Kind, red.Where, state, firstLine(red.Said))))
}
// The same item on several machines: a copy that is in use and far smaller than one kept retired elsewhere is an
// empty replacement. Only against a retired copy — a module running on two machines on purpose keeps two different
// sets of data.
items := make([]string, 0, len(byItem))
for k := range byItem {
items = append(items, k)
}
sort.Strings(items)
for _, key := range items {
for _, a := range byItem[key] {
if a.SizeBytes == nil || !catalogue.Watched(a.Class) || !comparable(a.Precision) {
continue
}
for _, o := range retiredCopies[key] {
if o.Node == a.Node || o.SizeBytes == nil || !comparable(o.Precision) || !replacedByLess(*a.SizeBytes, *o.SizeBytes) {
continue
}
ob := obs(kindEmptyReplacement, keyOfItem(a.Node, a.Module, a.ID), a.Node, severityOf(a.Class),
fmt.Sprintf("%s of %s on %s holds %s, and the copy %s kept on %s holds %s: %s is running on "+
"an empty replacement of its data. Move the data, or assign it back where its data is",
a.ID, a.Module, a.Node, sizeWords(a.SizeBytes), o.Module, o.Node, sizeWords(o.SizeBytes), a.Module), "")
ob.Subject.Also = []string{o.Node}
out = append(out, ob)
break
}
}
}
return out
}
// replacedByLess is whether a copy in use is an empty replacement of a copy kept elsewhere: less than half of it, and at
// least shrinkFloor less.
func replacedByLess(inUse, kept int64) bool { return inUse*2 < kept && kept-inUse >= shrinkFloor }
modules/mesh-self-check/cmd/mesh-self-check/probes_test.go
+27
View File
@@ -187,6 +187,22 @@ func TestTheBusProbes(t *testing.T) {
r.size("ace", "plex", "thumbs", 10<<20, 600<<20).dataset("ace", "plex", "thumbs", "storage/media")
r.size("ace", "plex", "meta", 10<<20, 600<<20).dataset("ace", "plex", "meta", "storage/media")
})},
{"D13 a retired item is kept, not judged, and cleanup waits after thirty days", probeData, bus{}, []string{
"machine.ace.pg.dumps.cleanup"}, "", dataMesh(func(r *dataRig) {
r.size("ace", "pg", "dumps", 1<<20, 1<<30).retired("ace", "pg", "dumps", -31*24*time.Hour)
r.size("ace", "pg", "old", 1<<20, 1<<30).retired("ace", "pg", "old", -2*24*time.Hour)
})},
{"D13 a copy far smaller than a retired one elsewhere is an empty replacement", probeData, bus{},
[]string{"machine.ace.plex.thumbs.empty-replacement"}, "", dataMesh(func(r *dataRig) {
r.size("ace", "plex", "thumbs", 10<<20, 10<<20)
r.size("g14", "plex", "thumbs", 600<<20, 600<<20).retired("g14", "plex", "thumbs", -24*time.Hour)
})},
{"D13 no good backup is counted from when the item was first measured", probeData, bus{},
[]string{"machine.ace.pg.data.backup-stale"}, "", dataMesh(func(r *dataRig) {
delete(r.good, health.ReadingKey("ace", "pg", "data"))
delete(r.good, health.ReadingKey("ace", "plex", "meta"))
r.firstSeen("ace", "pg", "data", -100*time.Hour).firstSeen("ace", "plex", "meta", -time.Hour)
})},
{"D13 a machine with a backup holder and no reading", probeData, bus{}, []string{"machine.g14.data-unmeasured"}, "",
dataMesh(func(r *dataRig) { r.holder("g14") })},
// D-fill (ADR 0297 §6): a module's bucket or log at three quarters of its cap, kept until under seven tenths.
@@ -375,6 +391,17 @@ func (r *dataRig) lastGood(node, module, item string, ago time.Duration) *dataRi
return r
}
func (r *dataRig) retired(node, module, item string, ago time.Duration) *dataRig {
x := r.reading(node, module, item)
x.RetiredAt, x.RetiredWhy = dataNow.Add(ago), "no longer declared on "+node
return r
}
func (r *dataRig) firstSeen(node, module, item string, ago time.Duration) *dataRig {
r.reading(node, module, item).FirstSeen = dataNow.Add(ago)
return r
}
func (r *dataRig) holder(node string) {
key, _ := delivery.SeatHoldersKey("node-backup", node)
_ = r.s.Put(SeatHoldersRead, key, delivery.SeatHolders{V: 1, Seat: "node-backup", Scope: "node", Node: node,
modules/mesh-self-check/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-self-check
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029224808-f3677eb6f36e
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029225606-31e90aaa0f0d
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-self-check/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029224808-f3677eb6f36e h1:h8bh7ZyAgPuFC9/lc6iFBD/hduzomLodZ0a1oJZ5X4U=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029224808-f3677eb6f36e/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029225606-31e90aaa0f0d h1:HHSXxRS+GEYNgtdg9MFB8W2Y6ycPJF7YLF46y3oj7Ow=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029225606-31e90aaa0f0d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=