From 52af210e47deeaf51210a12c6883946a5172c492 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 6 Oct 2026 16:47:49 +0200 Subject: [PATCH 1/2] Derive data protection from a module's declared data (hq ADR 0233) A module's data section says what it keeps and how precious it is; the backup holder's lines, binding stickiness, retirement on unassign and D13's conditions follow from it, so issue 273's empty replacement is said and an unassigned module's data is remembered, not forgotten. --- cmd/mesh-controller/acts.go | 4 + cmd/mesh-controller/check.go | 17 + cmd/mesh-controller/data.go | 883 ++++++++++++++++++ cmd/mesh-controller/data_test.go | 430 +++++++++ cmd/mesh-controller/doctor.go | 8 + cmd/mesh-controller/main.go | 3 + cmd/mesh-controller/retire_verbs.go | 53 +- cmd/mesh-controller/retirement_test.go | 4 +- cmd/mesh-controller/seatverbs.go | 11 +- cmd/mesh-controller/seatverbs_schema_test.go | 1 + internal/broker/nats.go | 6 +- internal/broker/testdata/composed.conf | 2 +- internal/catalogue/backup_test.go | 41 +- internal/catalogue/catalogue_check_test.go | 29 +- internal/catalogue/data.go | 766 +++++++++++++++ internal/catalogue/data_test.go | 226 +++++ internal/catalogue/environment_into.go | 2 +- internal/catalogue/manifest.go | 26 +- internal/catalogue/seat_contributions.go | 69 +- internal/catalogue/seat_dependencies.go | 6 + internal/catalogue/seats.go | 6 +- internal/catalogue/verbs.go | 15 +- internal/inventory/data.go | 294 ++++++ internal/inventory/data_test.go | 109 +++ ...72-a-module-declares-the-data-it-holds.sql | 65 ++ internal/link/retirement.go | 11 +- 26 files changed, 2957 insertions(+), 130 deletions(-) create mode 100644 cmd/mesh-controller/data.go create mode 100644 cmd/mesh-controller/data_test.go create mode 100644 internal/catalogue/data.go create mode 100644 internal/catalogue/data_test.go create mode 100644 internal/inventory/data.go create mode 100644 internal/inventory/data_test.go create mode 100644 internal/inventory/migrations/0072-a-module-declares-the-data-it-holds.sql diff --git a/cmd/mesh-controller/acts.go b/cmd/mesh-controller/acts.go index bcd4626..5ad5f70 100644 --- a/cmd/mesh-controller/acts.go +++ b/cmd/mesh-controller/acts.go @@ -224,6 +224,10 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string) for _, line := range unheldChange(shelf, node, assigned, left) { answer += "\n " + line } + // What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233). + for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) { + answer += "\n " + line + } return answer + blockedElsewhere(ctx, open, node), nil } diff --git a/cmd/mesh-controller/check.go b/cmd/mesh-controller/check.go index fe1ed88..463af4d 100644 --- a/cmd/mesh-controller/check.go +++ b/cmd/mesh-controller/check.go @@ -93,6 +93,15 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { } failed += len(identities) + // And the data each module keeps (novox/hq ADR 0233): a provider that grants says what it keeps for + // its consumers, a directory a container writes is declared, and no backup line is written by hand. + data := catalogue.DataProblems(shelf) + sort.Strings(data) + for _, p := range data { + fmt.Fprintln(out, p) + } + failed += len(data) + var names []string for name := range shelf { names = append(names, name) @@ -121,6 +130,14 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { if len(m.Reads) > 0 { fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", ")) } + // The data it keeps, by class, so a reviewer sees what the mesh will protect and how. + if items := m.DataItems(); len(items) > 0 { + kept := make([]string, 0, len(items)) + for _, it := range items { + kept = append(kept, it.ID+" ("+it.Class+")") + } + fmt.Fprintf(out, ", keeps %s", strings.Join(kept, ", ")) + } fmt.Fprintln(out) } if failed > 0 { diff --git a/cmd/mesh-controller/data.go b/cmd/mesh-controller/data.go new file mode 100644 index 0000000..874e477 --- /dev/null +++ b/cmd/mesh-controller/data.go @@ -0,0 +1,883 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "log" + "sort" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A module declares the data it holds, and the mesh protects and watches it from that declaration +// (novox/hq ADR 0233). +// +// The self-check's D13 composes what every machine declares, asks each machine's backup holder what +// it measured of every item — size, newest write, newest good backup, the redundant storage it is on — +// and keeps both. From that, and from what every provider says it holds for its consumers, it raises, +// each URGENT for what is irreplaceable and a WARNING for what is valuable (the operator's ranking): +// +// - `data-shrank`: an item holds less than half of its largest size in seven days, and at least +// shrinkFloor less; `data-missing`: its path is gone; +// - `empty-replacement`: an item, or a consumer's data at a provider, is less than half the size of a +// copy of the same thing kept elsewhere — on 2026-10-05 five applications ran for twenty hours on +// empty databases while their real ones sat on another machine (issue 273); +// - `data-held-twice` (warning): a consumer has active data at two providers and their sizes cannot +// be compared; +// - `data-quiet`: an item said to be written all the time has not been, within its bound; +// - `backup-stale`: an item's newest good backup is older than its bound, or there is none; +// - `array-degraded`: the redundant storage an item is on is not healthy, or cannot be read; +// `protection-missing`: an item said to be protected by redundancy is on storage that is not; +// - `cleanup-waiting` (warning): an item retired more than thirty days, waiting for a person. +// +// And it retires: an irreplaceable or valuable item in a module's own directory that its machine no +// longer declares — its module unassigned — is kept, marked retired with when and why, and listed by +// `cleanup list` until `cleanup delete` removes it. The node-engine never deletes a directory with +// anything in it; this is the record of what it kept. An operator's path is never retired or deleted. + +// The condition kinds of D13. +const ( + kindDataShrank = "data-shrank" + kindEmptyReplacement = "empty-replacement" + kindDataHeldTwice = "data-held-twice" + kindDataQuiet = "data-quiet" + kindBackupStale = "backup-stale" + kindDataUnmeasured = "data-unmeasured" + // kindDataMissing is a watched item whose path is gone. + kindDataMissing = "data-missing" + // kindArrayDegraded is redundant storage watched data is on that is not healthy, or cannot be read. + kindArrayDegraded = "array-degraded" + // kindProtectionMissing is an item said to be protected by redundancy, on storage that is not. + kindProtectionMissing = "protection-missing" +) + +// probeDataID is the self-check's id for this probe. +const probeDataID = "D13" + +// The bounds the findings are read against. +var ( + // shrinkWindow is how far back the largest size is looked for. + shrinkWindow = 7 * 24 * time.Hour + // shrinkFloor is the least loss that is worth saying: two empty databases differ by a few + // megabytes, and half of almost nothing is noise. + shrinkFloor int64 = 16 << 20 + // dataAsk is how long one machine's holder, or one provider, is given to answer. + dataAsk = 8 * time.Second +) + +// keyOfItem is one item's condition id: its machine, module and item. +func keyOfItem(machine, module, item string) string { return machine + "." + module + "." + item } + +// holderAnswer is what a node-backup holder's `backed-up` says of one module (ADR 0233 adds Data). +type holderAnswer struct { + Module string `json:"module"` + Data []holderItem `json:"data"` +} + +// holderItem is one item as the holder measured it. +type holderItem struct { + Item string `json:"item"` + Class string `json:"class"` + Path string `json:"path"` + SizeBytes *int64 `json:"size_bytes"` + LastWrite *time.Time `json:"last_write"` + MeasuredAt *time.Time `json:"measured_at"` + LastBackup *time.Time `json:"last_backup"` + Error string `json:"error,omitempty"` + // Redundancy is the redundant storage the item is on, where the holder could tell (ADR 0233). + Redundancy *inventory.Redundancy `json:"redundancy,omitempty"` +} + +// readHolder reads a holder's answer into measurements by module and item. +func readHolder(raw json.RawMessage) (map[string]map[string]inventory.Measurement, error) { + var modules []holderAnswer + if err := json.Unmarshal(raw, &modules); err != nil { + return nil, fmt.Errorf("its answer is not readable: %w", err) + } + out := map[string]map[string]inventory.Measurement{} + for _, m := range modules { + for _, it := range m.Data { + if out[m.Module] == nil { + out[m.Module] = map[string]inventory.Measurement{} + } + out[m.Module][it.Item] = inventory.Measurement{Path: it.Path, Size: it.SizeBytes, LastWrite: it.LastWrite, + MeasuredAt: it.MeasuredAt, LastBackup: it.LastBackup, Error: it.Error, Redundancy: it.Redundancy} + } + } + return out, nil +} + +// declaredOn is every data item a machine's composition declares, and whether something there holds +// node-backup to measure them. +func declaredOn(plan catalogue.Resolution) ([]inventory.DeclaredData, bool) { + var out []inventory.DeclaredData + held := false + for _, m := range plan.Modules { + for _, c := range m.Claims { + if s, known := catalogue.SeatNamed(c.Name); known && s.Name == catalogue.BackupSeat { + held = true + } + } + for _, it := range m.DataItems() { + out = append(out, inventory.DeclaredData{Module: m.Module, Item: it.ID, Class: it.Class, + Owned: it.OwnedByModule(), Protection: it.Protection()}) + } + } + return out, held +} + +// consumerCopy is one provider's account of one consumer: where, how big, and whether still active. +type consumerCopy struct { + Node, Module, Consumer string + Size *int64 + Retired bool + // Class is how precious the consumer's data is: the stricter of what the provider keeps for its + // consumers and what the consumer says it keeps there (`kept-by`). + Class string +} + +// probeData is D13. +func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error) { + if d.js == nil { + return nil, errors.New("no bus to ask the machines over") + } + open := d.open + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + return nil, err + } + nodes, err := open.inventory.Nodes(ctx) + if err != nil { + return nil, err + } + heard := heardMachines(d) + now := time.Now() + + type machine struct { + name string + declared []inventory.DeclaredData + held bool + measured map[string]map[string]inventory.Measurement + askErr error + } + var machines []*machine + for _, n := range nodes { + plan, _, err := planFor(ctx, open, n.Name) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + // A machine that cannot be worked out declares nothing this run — which is not the same as + // declaring nothing: retiring its data on that would be acting on an unreadable result. + continue + } + declared, held := declaredOn(plan) + machines = append(machines, &machine{name: n.Name, declared: declared, held: held}) + } + // Every holder asked at once, as D8 asks every ban list. + var wg sync.WaitGroup + for _, m := range machines { + if !m.held || !heard[m.name] { + continue + } + wg.Add(1) + go func(m *machine) { + defer wg.Done() + asking, cancel := context.WithTimeout(ctx, dataAsk) + defer cancel() + raw, err := askSeatTool(asking, d.js.Conn(), catalogue.BackupSeat, "backed-up", m.name) + if err == nil { + m.measured, err = readHolder(raw) + } + m.askErr = err + }(m) + } + wg.Wait() + + var out []conditions.Observation + for _, m := range machines { + if m.askErr != nil { + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Token: kindDataUnmeasured, + Kind: kindDataUnmeasured, Machine: m.name, Severity: conditions.Warning, + Summary: fmt.Sprintf("%s's backup holder did not say what it measured of the data declared there, so "+ + "nothing about that data is known this run: %s", m.name, firstLine(m.askErr.Error())), + Said: firstLine(m.askErr.Error())}) + } + why := fmt.Sprintf("no longer declared on %s: its module was unassigned there, or is no longer pulled in", m.name) + change, err := open.inventory.RecordData(ctx, m.name, m.declared, m.measured, why, now) + if err != nil { + return nil, fmt.Errorf("what %s holds could not be kept: %w", m.name, err) + } + for _, r := range change.Retired { + log.Printf("data: %s of %s on %s RETIRED, kept at %s: %s — `cleanup list` shows it, and only `cleanup "+ + "delete` removes it (novox/hq ADR 0233)", r.Item, r.Module, r.Machine, orUnknownPath(r.Path), why) + } + for _, r := range change.Reenabled { + log.Printf("data: %s of %s on %s is declared again, no longer retired", r.Item, r.Module, r.Machine) + } + } + + records, err := open.inventory.Data(ctx) + if err != nil { + return nil, err + } + peaks, err := open.inventory.DataPeaks(ctx, now.Add(-shrinkWindow)) + if err != nil { + return nil, err + } + bindings, err := open.inventory.Bindings(ctx) + if err != nil { + return nil, err + } + upgraded, keptBy := keptByClasses(bindings, shelf) + copies, err := consumerCopies(ctx, d.js.Conn(), open.inventory, shelf, keptBy) + if err != nil { + return nil, err + } + out = append(out, dataFindings(records, peaks, shelf, copies, upgraded, now)...) + return out, nil +} + +func orUnknownPath(p string) string { + if p == "" { + return "a path its backup holder never named" + } + return p +} + +// consumerCopies asks every provider of a provision whose consumers' data is kept what it holds, at +// once. One that cannot answer is passed over: it says nothing about any copy, which is not a finding. +func consumerCopies(ctx context.Context, conn *nats.Conn, inv *inventory.Inventory, + shelf map[string]catalogue.Manifest, keptBy map[string]string) ([]consumerCopy, error) { + instances, err := providerInstances(ctx, inv) + if err != nil { + return nil, err + } + var asked []providerInstance + for _, p := range instances { + m := shelf[p.Module] + keeps := false + for provision := range m.Grants { + keeps = keeps || m.KeepsConsumerData(provision) + } + if keeps { + asked = append(asked, p) + } + } + states := make([]*link.RetirementState, len(asked)) + var wg sync.WaitGroup + for i, p := range asked { + wg.Add(1) + go func(i int, p providerInstance) { + defer wg.Done() + asking, cancel := context.WithTimeout(ctx, dataAsk) + defer cancel() + if s, err := askRetirement(asking, conn, p); err == nil { + states[i] = &s + } + }(i, p) + } + wg.Wait() + var out []consumerCopy + for i, p := range asked { + s := states[i] + if s == nil { + continue + } + class := consumersClass(shelf[p.Module]) + for _, c := range s.Held { + cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: c, + Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+c])} + if size, ok := s.HeldSizes[c]; ok && size >= 0 { + size := size + cp.Size = &size + } + out = append(out, cp) + } + for _, r := range s.Retired { + if r.Kind != "" && r.Kind != "consumer" { + continue + } + cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: r.Consumer, Retired: true, + Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+r.Consumer])} + if r.SizeBytes != nil && *r.SizeBytes >= 0 { + cp.Size = r.SizeBytes + } + out = append(out, cp) + } + } + return out, nil +} + +// severityOf is how loud a finding about data of a class is: urgent for what is irreplaceable, a warning +// for anything else watched (the operator's ranking, ADR 0233). +func severityOf(class string) conditions.Severity { + if class == catalogue.ClassIrreplaceable { + return conditions.Urgent + } + return conditions.Warning +} + +// consumersClass is the most precious class a provider keeps any of its consumers' data as. +func consumersClass(m catalogue.Manifest) string { + class := catalogue.ClassNone + for provision := range m.Grants { + if c, ok := m.ConsumerDataOf(provision); ok { + class = catalogue.StricterClass(class, c.Class) + } else if m.KeepsConsumerData(provision) { + class = catalogue.StricterClass(class, catalogue.ClassValuable) + } + } + return class +} + +// keptByClasses is what consumers say of the data they keep with their providers (`kept-by`), read +// through where each is bound: by provider module and consumer identity, the class of that consumer's +// data there; and by provider item key (machine/module/item), the class the item holding it is held to. +func keptByClasses(bindings []inventory.Binding, shelf map[string]catalogue.Manifest) (map[string]string, map[string]string) { + upgraded, keptBy := map[string]string{}, map[string]string{} + for _, b := range bindings { + m, ok := shelf[b.Consumer] + if !ok { + continue + } + k, said := m.KeptByOf(b.Provision) + if !said { + continue + } + identity := catalogue.ConsumerIdentity(b.Machine, catalogue.IdentitySource(m.Slug, m.Module)) + key := b.Provider.Module + "/" + identity + keptBy[key] = catalogue.StricterClass(keptBy[key], k.Class) + if pc, ok := shelf[b.Provider.Module].ConsumerDataOf(b.Provision); ok && pc.In != "" { + if _, own := shelf[b.Provider.Module].DataItem(pc.In); own { + item := b.Provider.Node + "/" + b.Provider.Module + "/" + pc.In + upgraded[item] = catalogue.StricterClass(upgraded[item], k.Class) + } + } + } + return upgraded, keptBy +} + +// dataFindings is every condition the data on record raises now. A function of what is known, so the +// incident's shape is tested without a mesh. upgraded is the class an item is held to where a consumer +// of its module keeps data in it more precious than its own class says (`kept-by`), by its key. +func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf map[string]catalogue.Manifest, + copies []consumerCopy, upgraded map[string]string, now time.Time) []conditions.Observation { + var out []conditions.Observation + byItem := map[string][]inventory.DataRecord{} + arrays := map[string][]inventory.DataRecord{} + for _, r := range records { + if r.DeletedAt != nil { + continue + } + class := catalogue.StricterClass(r.Class, upgraded[r.Key()]) + r.Class = class + byItem[r.Module+"/"+r.Item] = append(byItem[r.Module+"/"+r.Item], r) + item, declared := shelf[r.Module].DataItem(r.Item) + id := keyOfItem(r.Machine, r.Module, r.Item) + if r.Retired() { + if now.Sub(*r.RetiredAt) > cleanupAfter { + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "cleanup", + Kind: kindCleanupWaiting, Machine: r.Machine, Severity: conditions.Warning, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept at %s since %s; `cleanup "+ + "delete %s %s %s --why …` once a person has decided, or assign %s there again", + r.Item, r.Module, r.Machine, r.Class, sizeWords(r.Size), int(now.Sub(*r.RetiredAt).Hours()/24), + orUnknownPath(r.Path), r.RetiredWhy, r.Machine, r.Module, r.Item, r.Module)}) + } + continue + } + if !catalogue.Watched(class) { + continue + } + severity := severityOf(class) + if r.Redundancy != nil { + where := r.Machine + "/" + r.Redundancy.Kind + ":" + r.Redundancy.Where + arrays[where] = append(arrays[where], r) + } else if declared && item.Redundancy != "" && r.MeasuredAt != nil && r.MeasureError == "" { + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindProtectionMissing, + Kind: kindProtectionMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s of %s on %s (%s) is said to be protected by the redundancy of the storage it is on, "+ + "and %s is on nothing the backup holder can read as redundant: it has no protection the mesh can see", + r.Item, r.Module, r.Machine, class, orUnknownPath(r.Path))}) + } + if r.MeasureError != "" && strings.Contains(r.MeasureError, "does not exist") { + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataMissing, + Kind: kindDataMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s of %s on %s (%s) is gone: %s does not exist any more", r.Item, r.Module, r.Machine, + class, orUnknownPath(r.Path))}) + } else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && *r.Size*2 < peak && peak-*r.Size >= shrinkFloor { + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataShrank, + Kind: kindDataShrank, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s of %s on %s (%s) shrank to %s from %s within %d days — more than half of what it "+ + "held is gone. If that was meant, silence this with why; if not, `node-backup.restore` puts the last "+ + "good copy beside it", r.Item, r.Module, r.Machine, class, sizeWords(r.Size), sizeWords(&peak), + int(shrinkWindow.Hours()/24))}) + } + if !declared { + continue + } + if within := item.ActiveWithin(); within > 0 && r.LastWrite != nil && now.Sub(*r.LastWrite) > within { + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataQuiet, + Kind: kindDataQuiet, Machine: r.Machine, Severity: severity, + Summary: fmt.Sprintf("%s of %s on %s is written all the time, and has not been since %s (its bound is %s): "+ + "whatever writes it has stopped", r.Item, r.Module, r.Machine, r.LastWrite.UTC().Format(time.RFC3339), + within)}) + } + // A backup is required of what is irreplaceable and copied; of what is valuable it is the standard + // plan, said only where the machine was measured — where a holder is there to take it. + if item.BackedUp() && (class == catalogue.ClassIrreplaceable || r.MeasuredAt != nil) { + within := item.BackupWithin() + switch { + case r.LastBackup == nil && now.Sub(r.FirstSeen) > within: + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale, + Kind: kindBackupStale, Machine: r.Machine, Severity: severity, + Summary: fmt.Sprintf("%s of %s on %s is %s and has no good backup on record, %s after it was first "+ + "declared — is node-backup held there, and do its nights succeed? (`node-backup.backed-up`)", + r.Item, r.Module, r.Machine, class, now.Sub(r.FirstSeen).Round(time.Hour))}) + case r.LastBackup != nil && now.Sub(*r.LastBackup) > within: + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale, + Kind: kindBackupStale, Machine: r.Machine, Severity: severity, + Summary: fmt.Sprintf("%s of %s on %s is %s and its newest good backup is from %s, older than its bound "+ + "of %s", r.Item, r.Module, r.Machine, class, r.LastBackup.UTC().Format(time.RFC3339), within)}) + } + } + } + // The redundant storage watched data is on: one condition per array, as loud as the most precious + // item on it — the array, not each item, is what degrades. + for _, where := range keysSorted(arrays) { + rs := arrays[where] + red := rs[0].Redundancy + if red.Healthy != nil && *red.Healthy { + continue + } + class, machine := catalogue.ClassValuable, rs[0].Machine + var names []string + for _, r := range rs { + class = catalogue.StricterClass(class, r.Class) + names = append(names, r.Module+"/"+r.Item) + } + state := "could not be read" + if red.Healthy != nil { + state = "is NOT healthy" + } + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, + ID: machine + ".array." + strings.NewReplacer("/", "-", ":", "-").Replace(red.Kind+"-"+red.Where), + Token: kindArrayDegraded, Kind: kindArrayDegraded, Machine: machine, Severity: severityOf(class), + Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("the %s storage %s on %s %s: %s — and it is what protects %s", red.Kind, red.Where, machine, + state, firstLine(red.Said), strings.Join(names, ", "))}) + } + // The same item on several machines: a copy that is in use and far smaller than one kept elsewhere is + // an empty replacement. Only against a retired copy — a module running on two machines on purpose + // keeps two different sets of data. + for _, key := range keysSorted(byItem) { + rs := byItem[key] + for _, a := range rs { + if a.Retired() || a.Size == nil || !catalogue.Watched(a.Class) { + continue + } + for _, o := range rs { + if o.Machine == a.Machine || !o.Retired() || o.Size == nil || !replacedByLess(*a.Size, *o.Size) { + continue + } + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, + ID: keyOfItem(a.Machine, a.Module, a.Item), Token: kindEmptyReplacement, Kind: kindEmptyReplacement, + Machine: a.Machine, Also: []string{o.Machine}, Severity: severityOf(a.Class), Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s of %s on %s holds %s, and the copy %s kept on %s holds %s: %s is running on "+ + "an empty replacement of its data. Move the data, or assign it back where its data is", + a.Item, a.Module, a.Machine, sizeWords(a.Size), o.Module, o.Machine, sizeWords(o.Size), a.Module)}) + break + } + } + } + out = append(out, consumerFindings(copies)...) + return out +} + +// replacedByLess is whether a copy in use is an empty replacement of a copy kept elsewhere: less than +// half of it, and at least shrinkFloor less. +func replacedByLess(inUse, kept int64) bool { + return inUse*2 < kept && kept-inUse >= shrinkFloor +} + +// consumerFindings is the same question of consumers' data at providers: one consumer, the same +// provider module on two machines. +func consumerFindings(copies []consumerCopy) []conditions.Observation { + by := map[string][]consumerCopy{} + for _, c := range copies { + k := c.Module + "/" + c.Consumer + by[k] = append(by[k], c) + } + var out []conditions.Observation + for _, k := range keysSorted(by) { + cs := by[k] + if len(cs) < 2 { + continue + } + found := false + for _, a := range cs { + if a.Retired || a.Size == nil { + continue + } + for _, o := range cs { + if o.Node == a.Node || o.Size == nil || !replacedByLess(*a.Size, *o.Size) { + continue + } + state := "active" + if o.Retired { + state = "retired" + } + out = append(out, conditions.Observation{Scope: conditions.ScopeProvider, + ID: a.Module + "." + a.Node + "." + a.Consumer, Token: kindEmptyReplacement, Kind: kindEmptyReplacement, + Machine: a.Node, Also: []string{o.Node}, Severity: severityOf(catalogue.StricterClass(a.Class, o.Class)), + Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s's data at %s on %s holds %s, and its %s copy at %s on %s holds %s: the "+ + "consumer is using an empty replacement of its data (issue 273's shape). Pin it back to %s, or move "+ + "the data first", a.Consumer, a.Module, a.Node, sizeWords(a.Size), state, o.Module, o.Node, + sizeWords(o.Size), o.Node)}) + found = true + break + } + if found { + break + } + } + if found { + continue + } + var active []consumerCopy + for _, c := range cs { + if !c.Retired { + active = append(active, c) + } + } + if len(active) >= 2 { + var where []string + var also []string + for _, c := range active { + where = append(where, c.Node+" ("+sizeWords(c.Size)+")") + also = append(also, c.Node) + } + out = append(out, conditions.Observation{Scope: conditions.ScopeProvider, + ID: active[0].Module + "." + active[0].Consumer, Token: kindDataHeldTwice, Kind: kindDataHeldTwice, + Machine: active[0].Node, Also: also[1:], Severity: conditions.Warning, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s has active data at %s on %d machines — %s — and only one is the one it uses", + active[0].Consumer, active[0].Module, len(active), strings.Join(where, ", "))}) + } + } + return out +} + +func keysSorted[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// ---- the `data` verb --------------------------------------------------------------------------- + +const dataUsage = "data [--json] [--machine ] [--retired]" + +// dataRow is one item as `data` lists it. +type dataRow struct { + Machine string `json:"machine"` + Module string `json:"module"` + Item string `json:"item"` + Class string `json:"class"` + Path string `json:"path,omitempty"` + Protection string `json:"protection,omitempty"` + // Array is the redundant storage it is on and its state, where its holder could tell. + Array string `json:"array,omitempty"` + Unmeasured string `json:"unmeasured,omitempty"` + SizeBytes *int64 `json:"size-bytes,omitempty"` + LastWrite string `json:"last-write,omitempty"` + MeasuredAt string `json:"measured-at,omitempty"` + LastBackup string `json:"last-backup,omitempty"` + BackupDue string `json:"backup-within,omitempty"` + Retired string `json:"retired,omitempty"` + RetiredWhy string `json:"retired-why,omitempty"` + Deleted string `json:"deleted,omitempty"` +} + +// dataCommand is `data`: every item every machine declares, or held retired, as the self-check last +// found it. +func dataCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("data", flag.ContinueOnError) + asJSON := set.Bool("json", false, "as data") + only := set.String("machine", "", "one machine") + retiredOnly := set.Bool("retired", false, "only what is retired") + if rest, err := parseAround(set, args); err != nil { + return err + } else if len(rest) > 0 { + return errors.New(dataUsage) + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + records, err := open.inventory.Data(ctx) + if err != nil { + return err + } + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + return err + } + rows := dataRows(records, shelf, *only, *retiredOnly) + if *asJSON { + return printJSON(map[string]any{"data": rows}) + } + if len(rows) == 0 { + fmt.Println("no data on record: the self-check (D13) records what each machine declares on its next run") + return nil + } + for _, r := range rows { + state := "" + switch { + case r.Deleted != "": + state = " DELETED " + r.Deleted + case r.Retired != "": + state = " RETIRED " + r.Retired + " — " + r.RetiredWhy + } + fmt.Printf("%s %s/%s %s %s %s protected by %s%s\n", r.Machine, r.Module, r.Item, r.Class, + sizeWords(r.SizeBytes), orUnknownPath(r.Path), orNothingWord(r.Protection), state) + if r.Array != "" { + fmt.Printf(" on %s\n", r.Array) + } + if r.Unmeasured != "" { + fmt.Printf(" not measured: %s\n", r.Unmeasured) + } + if r.Class == catalogue.ClassCache { + continue + } + fmt.Printf(" last write %s, measured %s, last backup %s%s\n", orNever(r.LastWrite), orNever(r.MeasuredAt), + orNever(r.LastBackup), within(r.BackupDue)) + } + return nil +} + +func dataRows(records []inventory.DataRecord, shelf map[string]catalogue.Manifest, only string, retiredOnly bool) []dataRow { + rows := []dataRow{} + stamp := func(t *time.Time) string { + if t == nil { + return "" + } + return t.UTC().Format(time.RFC3339) + } + for _, r := range records { + if only != "" && r.Machine != only { + continue + } + if retiredOnly && !r.Retired() { + continue + } + row := dataRow{Machine: r.Machine, Module: r.Module, Item: r.Item, Class: r.Class, Path: r.Path, + Protection: r.Protection, Unmeasured: r.MeasureError, SizeBytes: r.Size, LastWrite: stamp(r.LastWrite), MeasuredAt: stamp(r.MeasuredAt), + LastBackup: stamp(r.LastBackup), Retired: stamp(r.RetiredAt), RetiredWhy: r.RetiredWhy, + Deleted: stamp(r.DeletedAt)} + if it, ok := shelf[r.Module].DataItem(r.Item); ok && it.BackedUp() { + row.BackupDue = it.BackupWithin().String() + } + if red := r.Redundancy; red != nil { + state := "state unread" + if red.Healthy != nil && *red.Healthy { + state = "healthy" + } else if red.Healthy != nil { + state = "NOT HEALTHY" + } + row.Array = red.Kind + " " + red.Where + ", " + state + } + rows = append(rows, row) + } + return rows +} + +func orNothingWord(s string) string { + if s == "" || s == "none" { + return "nothing" + } + return s +} + +func orNever(s string) string { + if s == "" { + return "never" + } + return s +} + +func within(s string) string { + if s == "" { + return " (not backed up)" + } + return " (bound " + s + ")" +} + +// ---- cleanup of retired own data --------------------------------------------------------------- + +// The tools a node-backup holder serves to delete one retired item (novox/hq ADR 0233): the first +// takes a last restore point of it, tagged as retired, and only then removes it — in the background, +// because a large item outlasts any call — and the second says how that went. Module tools, not seat +// verbs: only the controller's `cleanup delete` calls them, as it calls a provider's provisioner_delete. +const ( + ToolDeleteRetired = "backup_delete_retired" + ToolDeletedOutcome = "backup_deleted" +) + +// deletionWait is how long `cleanup delete` follows a deletion before handing it back to the person. +var deletionWait = 8 * time.Minute + +// deletionPoll is how often it asks. +var deletionPoll = 5 * time.Second + +// deletion is a holder's account of one deletion. +type deletion struct { + Started bool `json:"started"` + Running bool `json:"running"` + Done bool `json:"done"` + OK bool `json:"ok"` + Snapshot string `json:"snapshot"` + Error string `json:"error"` +} + +// retiredData is every retired item on record, as `cleanup list` shows them. +func retiredData(records []inventory.DataRecord, now time.Time) []retiredRow { + var out []retiredRow + for _, r := range records { + if !r.Retired() { + continue + } + out = append(out, retiredRow{Node: r.Machine, Module: r.Module, Consumer: r.Item, Kind: retiredDataKind, + RetiredAt: r.RetiredAt.UTC().Format(time.RFC3339), AgeDays: int(now.Sub(*r.RetiredAt).Hours() / 24), + SizeBytes: r.Size, Why: r.RetiredWhy, Path: r.Path, Class: r.Class}) + } + return out +} + +// retiredDataKind is what `cleanup list` calls a module's own retired data, beside a provider's consumer. +const retiredDataKind = "own-data" + +// holderOn is the module holding node-backup on a machine. +func holderOn(ctx context.Context, inv *inventory.Inventory, machine string) (string, error) { + held, err := inv.Holdings(ctx) + if err != nil { + return "", err + } + for _, h := range held { + if s, known := catalogue.SeatNamed(h.Claim); known && s.Name == catalogue.BackupSeat && h.Node == machine { + return h.Module, nil + } + } + return "", fmt.Errorf("nothing holds %s on %s, and it is the backup holder that deletes retired data there "+ + "(after a last restore point)", catalogue.BackupSeat, machine) +} + +// deleteRetiredData has a machine's backup holder delete one retired item: never one declared now, and +// never one not retired. The holder takes a last restore point of it first, so the deletion can be +// undone until a person forgets that restore point; the record says deleted only once the holder says +// it is. +func deleteRetiredData(ctx context.Context, conn *nats.Conn, open *stores, r inventory.DataRecord, f handActFlags) error { + inv := open.inventory + if !r.Retired() { + return fmt.Errorf("%s of %s on %s is not retired — only retired data is deleted. Nothing was done", + r.Item, r.Module, r.Machine) + } + if r.Path == "" { + return fmt.Errorf("%s of %s on %s was never measured, so where it is was never said; nothing was deleted", + r.Item, r.Module, r.Machine) + } + if plan, _, err := planFor(ctx, open, r.Machine); err == nil { + for _, m := range plan.Modules { + if _, still := m.DataItem(r.Item); still && m.Module == r.Module { + return fmt.Errorf("%s runs on %s again and declares %s: it is not retired any more. Nothing was done", + r.Module, r.Machine, r.Item) + } + } + } + holder, err := holderOn(ctx, inv, r.Machine) + if err != nil { + return err + } + f.record(ctx, "cleanup delete", []string{r.Machine, r.Module, r.Item}) + args := map[string]any{"module": r.Module, "item": r.Item, "path": r.Path, "confirm": r.Item, + "why": strings.TrimSpace(*f.why), "by": link.Caller(), "via": link.ViaController} + ask := func(tool string) (deletion, error) { + var d deletion + answer, err := link.AskModuleToolOn(ctx, conn, holder, tool, r.Machine, args, 25*time.Second) + if err != nil { + return d, err + } + if answer.Error != "" { + return d, fmt.Errorf("%s on %s refused: %s", holder, r.Machine, answer.Error) + } + return d, unmarshalAnswer(answer, &d) + } + d, err := ask(ToolDeleteRetired) + if err != nil { + return err + } + for waited := time.Duration(0); !d.Done && waited < deletionWait; waited += deletionPoll { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(deletionPoll): + } + if d, err = ask(ToolDeletedOutcome); err != nil { + return err + } + } + switch { + case !d.Done: + fmt.Printf("%s on %s is still taking the last restore point of %s and deleting it; `cleanup list` keeps "+ + "showing it until the holder says it is done — the same `cleanup delete` again reads how it went\n", + holder, r.Machine, r.Path) + return nil + case !d.OK: + return fmt.Errorf("%s on %s did NOT delete %s: %s", holder, r.Machine, r.Path, d.Error) + } + if err := inv.MarkDataDeleted(ctx, r.Machine, r.Module, r.Item, link.Caller(), strings.TrimSpace(*f.why), time.Now()); err != nil { + return fmt.Errorf("%s deleted %s on %s, and it could not be recorded: %w", holder, r.Path, r.Machine, err) + } + fmt.Printf("%s on %s deleted %s of %s (%s, %s); its last restore point is %s, kept until a person forgets it\n", + holder, r.Machine, r.Item, r.Module, r.Path, sizeWords(r.Size), orNever(d.Snapshot)) + return nil +} + +// keptOnUnassign says, for an unassignment, the irreplaceable and valuable data each module leaves in its +// own directories on the machine: +// kept, and retired at the self-check's next run. +func keptOnUnassign(ctx context.Context, inv *inventory.Inventory, machine string, modules []string) []string { + records, err := inv.Data(ctx) + if err != nil { + return []string{"what it leaves behind could not be read from the mesh's record: " + err.Error()} + } + var out []string + for _, r := range records { + if r.Machine != machine || r.DeletedAt != nil || !catalogue.Retires(r.Class) || !r.Owned { + continue + } + for _, m := range modules { + if r.Module == m { + out = append(out, fmt.Sprintf("%s's %s (%s, %s) stays where it is: it is %s, so it is retired, "+ + "never removed — `cleanup list` shows it, `cleanup delete` alone removes it (novox/hq ADR 0233)", + r.Module, r.Item, orUnknownPath(r.Path), sizeWords(r.Size), r.Class)) + } + } + } + return out +} diff --git a/cmd/mesh-controller/data_test.go b/cmd/mesh-controller/data_test.go new file mode 100644 index 0000000..9fc3f00 --- /dev/null +++ b/cmd/mesh-controller/data_test.go @@ -0,0 +1,430 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/link" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +func bytesOf(n int64) *int64 { return &n } + +func when(t time.Time) *time.Time { return &t } + +func shelfFor(t *testing.T, manifests ...string) map[string]catalogue.Manifest { + t.Helper() + out := map[string]catalogue.Manifest{} + for _, raw := range manifests { + m, err := catalogue.ParseManifest([]byte(raw)) + if err != nil { + t.Fatal(err) + } + out[m.Module] = m + } + return out +} + +const houseManifest = `{"module":"house","version":"1", + "data":{"own":[{"id":"config","path":"${dir:config}","class":"irreplaceable","active":"1d"}]}, + "resources":[{"id":"config","type":"directory","mode":"0700"}]}` + +func findingsByKind(obs []conditions.Observation) map[string]conditions.Observation { + out := map[string]conditions.Observation{} + for _, o := range obs { + out[o.Kind] = o + } + return out +} + +// THE INCIDENT (issue 273), replayed against what D13 reads: five applications on the home server bound, +// by one changed rule, to the store on the control node, which made each an empty database — while +// their real databases, hundreds of megabytes each, sat on the home server's own store, by then retired +// because the mesh no longer asked for them there. Each is an empty replacement, naming both machines +// and the pin back: a warning for the store's consumers, whose data is valuable; urgent for one that says +// its data there is irreplaceable (`kept-by`). +func TestAnEmptyReplacementOfAConsumersDataIsSaid(t *testing.T) { + var copies []consumerCopy + for _, app := range []string{"mesh_home_board", "mesh_home_flows", "mesh_home_agents", "mesh_home_game", "mesh_home_cars"} { + copies = append(copies, + consumerCopy{Node: "home", Module: "postgres", Consumer: app, Size: bytesOf(400 << 20), Retired: true, Class: "valuable"}, + consumerCopy{Node: "anchor", Module: "postgres", Consumer: app, Size: bytesOf(9 << 20), Class: "valuable"}) + } + copies[1].Class = "irreplaceable" // the photo site's own database says so + got := dataFindings(nil, nil, nil, copies, nil, time.Now()) + if len(got) != 5 { + t.Fatalf("%d findings for five empty replacements: %+v", len(got), got) + } + for i, o := range got { + want := conditions.Warning + if strings.Contains(o.ID, "mesh_home_board") { + want = conditions.Urgent + } + _ = i + if o.Kind != kindEmptyReplacement || o.Severity != want || o.Machine != "anchor" || + len(o.Also) != 1 || o.Also[0] != "home" || !strings.Contains(o.Summary, "Pin it back to home") { + t.Errorf("%+v", o) + } + } + + // While the old copy is still active (the first ten minutes), it is the same finding. + copies[0].Retired = false + copies[1].Class = "valuable" + if got := dataFindings(nil, nil, nil, copies[:2], nil, time.Now()); len(got) != 1 || got[0].Kind != kindEmptyReplacement { + t.Fatalf("with the old copy still active: %+v", got) + } +} + +// A move a person made — the data moved first, then pinned — leaves a full copy at the new provider and +// a retired one at the old: nothing to say here; `cleanup` covers the old one. +func TestADeliberateMoveIsNoEmptyReplacement(t *testing.T) { + copies := []consumerCopy{ + {Node: "home", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(400 << 20), Retired: true}, + {Node: "anchor", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(402 << 20)}, + } + if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 { + t.Fatalf("a deliberate move raised %+v", got) + } + // Two small databases differing by less than the floor are not a finding either. + copies[0].Size, copies[1].Size = bytesOf(12<<20), bytesOf(8<<20) + if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 { + t.Fatalf("noise between two empty databases raised %+v", got) + } +} + +// Where a provider cannot say sizes, a consumer active at two providers is still said — as a warning, +// since which one is empty cannot be told. +func TestConsumerDataActiveTwiceWithoutSizesIsAWarning(t *testing.T) { + copies := []consumerCopy{ + {Node: "home", Module: "minio", Consumer: "mesh_home_photos"}, + {Node: "anchor", Module: "minio", Consumer: "mesh_home_photos"}, + } + got := dataFindings(nil, nil, nil, copies, nil, time.Now()) + if len(got) != 1 || got[0].Kind != kindDataHeldTwice || got[0].Severity != conditions.Warning { + t.Fatalf("%+v", got) + } +} + +// The same incident for a module's own data: a module unassigned from one machine and assigned on +// another starts over in an empty directory while its full one is kept, retired, where it was. +func TestAnEmptyReplacementOfAModulesOwnDataIsUrgent(t *testing.T) { + now := time.Now() + retired := now.Add(-time.Hour) + records := []inventory.DataRecord{ + {Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config", + Size: bytesOf(2 << 30), RetiredAt: &retired, FirstSeen: now.Add(-90 * 24 * time.Hour)}, + {Machine: "anchor", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config", + Size: bytesOf(1 << 20), FirstSeen: now.Add(-time.Hour), LastWrite: when(now)}, + } + got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now)) + o, ok := got[kindEmptyReplacement] + if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "home" { + t.Fatalf("%+v", got) + } + // The same of a valuable item is a warning. + records[0].Class, records[1].Class = "valuable", "valuable" + if o := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))[kindEmptyReplacement]; o.Severity != conditions.Warning { + t.Fatalf("a valuable empty replacement: %+v", o) + } + records[0].Class, records[1].Class = "irreplaceable", "irreplaceable" + // Two machines running a module on purpose, both active, keep two sets of data: nothing to say. + records[0].RetiredAt = nil + if got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now)); got[kindEmptyReplacement].Kind != "" { + t.Fatalf("two active copies were read as a replacement: %+v", got) + } +} + +// An irreplaceable item that lost more than half of its largest size in a week is urgent; a smaller loss, +// or a loss under the floor, is not a finding. +func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) { + now := time.Now() + r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", + Size: bytesOf(300 << 20), FirstSeen: now.Add(-30 * 24 * time.Hour), LastWrite: when(now), LastBackup: when(now)} + shelf := shelfFor(t, houseManifest) + o := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): 1 << 30}, shelf, nil, nil, now))[kindDataShrank] + if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "shrank") { + t.Fatalf("%+v", o) + } + for _, peak := range []int64{500 << 20, 20 << 20} { + if got := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): peak}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" { + t.Errorf("a peak of %d raised a shrink", peak) + } + } + small := r + small.Size = bytesOf(1 << 20) + if got := findingsByKind(dataFindings([]inventory.DataRecord{small}, map[string]int64{r.Key(): 10 << 20}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" { + t.Error("a loss under the floor raised a shrink") + } +} + +// Data said to be written all the time and not written; data with no backup or an old one — urgent when +// irreplaceable, a warning when valuable; and a new item given its bound before it is said. +func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) { + now := time.Now() + shelf := shelfFor(t, houseManifest) + r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", + Size: bytesOf(1 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), LastWrite: when(now.Add(-3 * 24 * time.Hour)), + LastBackup: when(now.Add(-72 * time.Hour))} + got := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now)) + if got[kindDataQuiet].Severity != conditions.Urgent || got[kindBackupStale].Severity != conditions.Urgent { + t.Fatalf("irreplaceable: %+v", got) + } + valuable := r + valuable.Class, valuable.MeasuredAt = "valuable", when(now) // measured: a holder is there to take its backup + if got := findingsByKind(dataFindings([]inventory.DataRecord{valuable}, nil, shelf, nil, nil, now)); got[kindDataQuiet].Severity != conditions.Warning || + got[kindBackupStale].Severity != conditions.Warning { + t.Fatalf("valuable: %+v", got) + } + never := r + never.LastBackup = nil + if o := findingsByKind(dataFindings([]inventory.DataRecord{never}, nil, shelf, nil, nil, now))[kindBackupStale]; !strings.Contains(o.Summary, "no good backup") { + t.Fatalf("never backed up: %+v", o) + } + fresh := never + fresh.FirstSeen, fresh.LastWrite = now.Add(-time.Hour), when(now) + if got := dataFindings([]inventory.DataRecord{fresh}, nil, shelf, nil, nil, now); len(got) != 0 { + t.Fatalf("an item declared an hour ago, before its first night, raised %+v", got) + } +} + +// An item retired more than thirty days waits for a person; less, it is only listed. +func TestRetiredDataWaitingThirtyDaysIsSaid(t *testing.T) { + now := time.Now() + old, recent := now.Add(-31*24*time.Hour), now.Add(-2*24*time.Hour) + records := []inventory.DataRecord{ + {Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &old}, + {Machine: "home", Module: "attic", Item: "boxes", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &recent}, + } + got := dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now) + if len(got) != 1 || got[0].Kind != kindCleanupWaiting || !strings.Contains(got[0].Summary, "cleanup delete home house config") { + t.Fatalf("%+v", got) + } + if rows := retiredData(records, now); len(rows) != 2 || rows[0].Kind != retiredDataKind { + t.Fatalf("cleanup list: %+v", rows) + } +} + +// The holder's answer reads into measurements, by module and item. +func TestTheHoldersAnswerIsRead(t *testing.T) { + raw := []byte(`[{"module":"postgres","runs":1,"paths":["/var/lib/mesh-store/dumps"],"lastNight":null,"restorePoints":3, + "data":[{"item":"store","class":"irreplaceable","path":"/var/lib/mesh-store","covered_by":"/var/lib/mesh-store/dumps", + "size_bytes":1073741824,"last_write":"2026-10-06T10:00:00Z","measured_at":"2026-10-06T10:05:00Z","last_backup":"2026-10-06T03:10:00Z"}]}]`) + got, err := readHolder(raw) + if err != nil { + t.Fatal(err) + } + m := got["postgres"]["store"] + if m.Path != "/var/lib/mesh-store" || m.Size == nil || *m.Size != 1<<30 || m.LastBackup == nil || m.MeasuredAt == nil { + t.Fatalf("%+v", m) + } + // An older holder, which says no data, reads as nothing measured rather than a failure. + if got, err := readHolder([]byte(`[{"module":"postgres","runs":1,"paths":[]}]`)); err != nil || len(got) != 0 { + t.Fatalf("%v, %v", got, err) + } +} + +// fakeHolder answers node-backup's `backed-up` on one machine over a real bus, with what it is told it +// measured. +type fakeHolder struct { + mu sync.Mutex + modules []map[string]any +} + +func (f *fakeHolder) set(modules ...map[string]any) { + f.mu.Lock() + defer f.mu.Unlock() + f.modules = modules +} + +func (f *fakeHolder) serve(t *testing.T, conn *nats.Conn, node string) { + t.Helper() + sub, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) { + f.mu.Lock() + defer f.mu.Unlock() + body, _ := json.Marshal(map[string]any{"result": f.modules, "error": "", "node": node}) + _ = m.Respond(body) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + if err := conn.Flush(); err != nil { + t.Fatal(err) + } +} + +func measuredHouse(path string, size int64, at time.Time) map[string]any { + return map[string]any{"module": "house", "runs": 0, "paths": []string{path}, "data": []map[string]any{{ + "item": "config", "class": "irreplaceable", "path": path, "covered_by": path, "size_bytes": size, + "last_write": at, "measured_at": at, "last_backup": at}}} +} + +// UNASSIGNING A MODULE WITH IRREPLACEABLE DATA KEEPS THE DATA, and assigning it elsewhere onto an empty +// directory is an empty replacement — through the real stores and a real bus. The unassignment says the +// data stays; the self-check's next run retires it (kept, listed by cleanup), and when the module comes +// up on another machine with an empty directory while the full one waits retired, that is urgent. +func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { + t.Fatal(err) + } + register(t, open, catalogue.Manifest{Module: "keeper", Version: "1", + Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}) + house, err := catalogue.ParseManifest([]byte(houseManifest)) + if err != nil { + t.Fatal(err) + } + register(t, open, house) + if _, err := assign(ctx, open, "laptop", "house"); err == nil { + t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up") + } + for _, node := range []string{"laptop", "anchor"} { + if _, err := assign(ctx, open, node, "keeper"); err != nil { + t.Fatal(err) + } + } + if _, err := assign(ctx, open, "laptop", "house"); err != nil { + t.Fatal(err) + } + + conn := onATestBus(t) + js, err := broker.Dial(os.Getenv("MESH_TEST_NATS")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + laptop, anchor := &fakeHolder{}, &fakeHolder{} + laptop.serve(t, conn, "laptop") + anchor.serve(t, conn, "anchor") + now := time.Now() + heard := &watchdogs{last: &signalFacts{now: now, machines: []machineFacts{ + {name: "laptop", lastHeard: now}, {name: "anchor", lastHeard: now}}}} + d := &doctor{open: open, js: js, watchdogs: heard} + var d13 probe + for _, p := range probeRegistry { + if p.ID == probeDataID { + d13 = p + } + } + run := func() []conditions.Observation { + t.Helper() + probing := context.WithValue(ctx, probeAsksKey{}, d13) + obs, err := probeData(probing, d) + if err != nil { + t.Fatal(err) + } + return obs + } + + laptop.set(measuredHouse("/var/lib/house/config", 2<<30, now)) + if obs := run(); len(obs) != 0 { + t.Fatalf("a measured, backed-up item raised %+v", obs) + } + r, err := inv.DataOf(ctx, "laptop", "house", "config") + if err != nil || r.Path != "/var/lib/house/config" || r.Size == nil || *r.Size != 2<<30 || r.LastBackup == nil { + t.Fatalf("what the holder measured was not kept: %+v, %v", r, err) + } + + said, err := unassign(ctx, open, "laptop", "house") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(said, "house's config (/var/lib/house/config, 2.0 GB) stays where it is") { + t.Fatalf("the unassignment does not say the data stays:\n%s", said) + } + laptop.set() + run() + r, err = inv.DataOf(ctx, "laptop", "house", "config") + if err != nil || !r.Retired() || r.Path != "/var/lib/house/config" { + t.Fatalf("unassigned, the irreplaceable item is not kept retired: %+v, %v", r, err) + } + records, _ := inv.Data(ctx) + if rows := retiredData(records, time.Now()); len(rows) != 1 || rows[0].Path != "/var/lib/house/config" { + t.Fatalf("cleanup list: %+v", rows) + } + + // Assigned on the anchor, onto an empty directory. + if _, err := assign(ctx, open, "anchor", "house"); err != nil { + t.Fatal(err) + } + anchor.set(measuredHouse("/var/lib/house/config", 300<<10, time.Now())) + obs := findingsByKind(run()) + o, ok := obs[kindEmptyReplacement] + if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "laptop" { + t.Fatalf("an empty replacement of a module's data was not urgent: %+v", obs) + } +} + +// Data on redundant storage: the array it is on is watched, one condition per array as loud as the most +// precious item on it; an item said to be on redundancy and found on plain storage is said; an item +// whose path is gone is said. +func TestTheArrayUnderDataIsWatched(t *testing.T) { + now := time.Now() + media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}], + "data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array, no room to copy"}, + {"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array, no room to copy"}]}}` + shelf := shelfFor(t, media) + sick := false + on := func(item string, healthy *bool) inventory.DataRecord { + return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable", Owned: false, + Path: "/tank/" + item, Size: bytesOf(40 << 40), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now), + Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: healthy, Said: "pool 'tank' is DEGRADED"}} + } + got := dataFindings([]inventory.DataRecord{on("films", &sick), on("shows", &sick)}, nil, shelf, nil, nil, now) + if len(got) != 1 || got[0].Kind != kindArrayDegraded || got[0].Severity != conditions.Urgent || + !strings.Contains(got[0].Summary, "media/films, media/shows") { + t.Fatalf("%+v", got) + } + well := true + if got := dataFindings([]inventory.DataRecord{on("films", &well)}, nil, shelf, nil, nil, now); len(got) != 0 { + t.Fatalf("a healthy array raised %+v", got) + } + plain := on("films", nil) + plain.Redundancy = nil + if o := findingsByKind(dataFindings([]inventory.DataRecord{plain}, nil, shelf, nil, nil, now))[kindProtectionMissing]; o.Severity != conditions.Urgent { + t.Fatalf("redundancy said and not found: %+v", o) + } + gone := on("films", &well) + gone.MeasureError, gone.Size = "/tank/films does not exist", bytesOf(0) + if o := findingsByKind(dataFindings([]inventory.DataRecord{gone}, map[string]int64{gone.Key(): 40 << 40}, shelf, nil, nil, now))[kindDataMissing]; o.Severity != conditions.Urgent { + t.Fatalf("a vanished library: %+v", o) + } +} + +// What a consumer keeps with its provider as irreplaceable holds the provider's item to that class: +// the photo site's objects make the object store's data an urgent matter. +func TestKeptByHoldsTheProvidersItemToTheConsumersClass(t *testing.T) { + objects := `{"module":"objects","version":"1","provides":[{"name":"s3-bucket","scope":"mesh"}],"grants":{"s3-bucket":"${dir:g}"}, + "data":{"own":[{"id":"data","path":"${dir:data}","class":"valuable"}],"consumers":{"s3-bucket":{"class":"valuable","in":"data"}}}, + "resources":[{"id":"g","type":"directory","mode":"0700"},{"id":"data","type":"directory","mode":"0700"}]}` + photos := `{"module":"photos","version":"1","requires":["s3-bucket"],"data":{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}}` + shelf := shelfFor(t, objects, photos) + bindings := []inventory.Binding{{Machine: "anchor", Consumer: "photos", Provision: "s3-bucket", + Provider: catalogue.Chosen{Node: "anchor", Module: "objects"}}} + upgraded, keptBy := keptByClasses(bindings, shelf) + if upgraded["anchor/objects/data"] != "irreplaceable" || keptBy["objects/mesh_anchor_photos"] != "irreplaceable" { + t.Fatalf("upgraded %v, kept by %v", upgraded, keptBy) + } + now := time.Now() + r := inventory.DataRecord{Machine: "anchor", Module: "objects", Item: "data", Class: "valuable", Owned: true, + Size: bytesOf(10 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), MeasuredAt: when(now), LastBackup: when(now.Add(-72 * time.Hour))} + if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, upgraded, now))[kindBackupStale]; o.Severity != conditions.Urgent { + t.Fatalf("the photos' store without a backup: %+v", o) + } + if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now))[kindBackupStale]; o.Severity != conditions.Warning { + t.Fatalf("a valuable store without a backup: %+v", o) + } +} diff --git a/cmd/mesh-controller/doctor.go b/cmd/mesh-controller/doctor.go index 310f31a..1394242 100644 --- a/cmd/mesh-controller/doctor.go +++ b/cmd/mesh-controller/doctor.go @@ -102,6 +102,14 @@ var probeRegistry = []probe{ {ID: probeBindingsID, Asserts: "every consumer of a provision that keeps its data is bound where it was last " + "sent, or moves by a pin", From: "issue 273, ADR 0232", Kind: kindBindingMoved, Raises: []string{kindBindingKept, kindBindingMoving}, Phase: 2, run: probeBindings}, + {ID: probeDataID, Asserts: "every item of data a machine declares is measured, is there, holds what it held, is " + + "written where it should be, is backed up within its bound or sits on healthy redundant storage, and is no " + + "empty replacement of a copy kept elsewhere; what a machine no longer declares that is irreplaceable or " + + "valuable is retired, not forgotten", From: "issue 273, ADR 0233", + Kind: kindDataShrank, Raises: []string{kindEmptyReplacement, kindDataHeldTwice, kindDataQuiet, kindBackupStale, + kindDataUnmeasured, kindDataMissing, kindArrayDegraded, kindProtectionMissing, kindCleanupWaiting}, + Phase: 2, run: probeData, + Asks: []broker.SeatVerb{{Seat: "node-backup", Verb: "backed-up"}}}, {ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals", From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs}, } diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index c6b7147..c63f756 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -170,6 +170,9 @@ func run() error { return retireCommand(ctx, args[1:]) case "cleanup": return cleanupCommand(ctx, args[1:]) + // The data every machine declares, as the self-check last found it (novox/hq ADR 0233). + case "data": + return dataCommand(ctx, args[1:]) case "version": fmt.Println(version) return nil diff --git a/cmd/mesh-controller/retire_verbs.go b/cmd/mesh-controller/retire_verbs.go index 30cb4e2..7b6368d 100644 --- a/cmd/mesh-controller/retire_verbs.go +++ b/cmd/mesh-controller/retire_verbs.go @@ -258,10 +258,21 @@ func cleanupCommand(ctx context.Context, args []string) error { } defer open.Close() return onTheBus(func(conn *nats.Conn) error { - return deleteOlderThan(ctx, open.inventory, conn, *olderThan, *confirm, f, time.Now()) + return deleteOlderThan(ctx, open, conn, *olderThan, *confirm, f, time.Now()) }) case *olderThan == 0 && len(rest) == 3 && !*confirm: + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() return onTheBus(func(conn *nats.Conn) error { + // A module's own retired data, when the mesh holds such an item (novox/hq ADR 0233); a + // provider's retired consumer otherwise. + if r, err := open.inventory.DataOf(ctx, rest[0], rest[1], rest[2]); err == nil && r.DeletedAt == nil && + r.RetiredAt != nil { + return deleteRetiredData(ctx, conn, open, r, f) + } return deleteRetired(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, rest[2], f) }) } @@ -285,6 +296,10 @@ type retiredRow struct { Why string `json:"why,omitempty"` // Access is "kept" for a consumer of a mark-only provider: retired on record, still reachable. Access string `json:"access,omitempty"` + // Path and Class are a module's own retired data's (Kind own-data, novox/hq ADR 0233): where it is + // kept on its machine, and its class. Consumer is then the item. + Path string `json:"path,omitempty"` + Class string `json:"class,omitempty"` } // retiredListing is every provider's retired consumers, and the providers that could not say. @@ -299,7 +314,15 @@ func gatherRetired(ctx context.Context, inv *inventory.Inventory, conn *nats.Con if err != nil { return retiredListing{}, err } - return retiredOf(ctx, conn, instances, now), nil + listing := retiredOf(ctx, conn, instances, now) + // And every module's own data retired on its machine (novox/hq ADR 0233). + records, err := inv.Data(ctx) + if err != nil { + return retiredListing{}, err + } + listing.Retired = append(listing.Retired, retiredData(records, now)...) + sort.SliceStable(listing.Retired, func(i, j int) bool { return listing.Retired[i].AgeDays > listing.Retired[j].AgeDays }) + return listing, nil } func retiredOf(ctx context.Context, conn *nats.Conn, instances []providerInstance, now time.Time) retiredListing { @@ -332,7 +355,7 @@ func printRetired(l retiredListing, asJSON bool) error { return printJSON(l) } if len(l.Retired) == 0 { - fmt.Println("no provider holds a retired consumer") + fmt.Println("no provider holds a retired consumer, and no machine holds retired data") } for _, r := range l.Retired { age := "age unknown" @@ -346,6 +369,11 @@ func printRetired(l retiredListing, asJSON bool) error { if r.Access == "kept" { kind += " [MARK ONLY: access kept until deleted]" } + if r.Kind == retiredDataKind { + fmt.Printf("%s on %s: its own %s, %s, at %s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer, + r.Class, r.Path, age, sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why)) + continue + } fmt.Printf("%s on %s: %s%s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer, kind, age, sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why)) } @@ -391,16 +419,16 @@ func deleteRetired(ctx context.Context, conn *nats.Conn, p providerInstance, con // deleteOlderThan lists every consumer retired more than days ago, and deletes them only with confirm. // One whose age the provider cannot say is never in it. -func deleteOlderThan(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, days int, confirm bool, +func deleteOlderThan(ctx context.Context, open *stores, conn *nats.Conn, days int, confirm bool, f handActFlags, now time.Time) error { - listing, err := gatherRetired(ctx, inv, conn, now) + listing, err := gatherRetired(ctx, open.inventory, conn, now) if err != nil { return err } - return deleteFrom(ctx, conn, listing, days, confirm, f) + return deleteFrom(ctx, conn, open, listing, days, confirm, f) } -func deleteFrom(ctx context.Context, conn *nats.Conn, listing retiredListing, days int, confirm bool, f handActFlags) error { +func deleteFrom(ctx context.Context, conn *nats.Conn, open *stores, listing retiredListing, days int, confirm bool, f handActFlags) error { var due []retiredRow unknown := 0 for _, r := range listing.Retired { @@ -431,7 +459,16 @@ func deleteFrom(ctx context.Context, conn *nats.Conn, listing retiredListing, da } var failed []string for _, r := range due { - if err := deleteRetired(ctx, conn, providerInstance{Node: r.Node, Module: r.Module}, r.Consumer, f); err != nil { + var err error + if r.Kind == retiredDataKind { + var rec inventory.DataRecord + if rec, err = open.inventory.DataOf(ctx, r.Node, r.Module, r.Consumer); err == nil { + err = deleteRetiredData(ctx, conn, open, rec, f) + } + } else { + err = deleteRetired(ctx, conn, providerInstance{Node: r.Node, Module: r.Module}, r.Consumer, f) + } + if err != nil { failed = append(failed, err.Error()) } } diff --git a/cmd/mesh-controller/retirement_test.go b/cmd/mesh-controller/retirement_test.go index 2784f09..e43912b 100644 --- a/cmd/mesh-controller/retirement_test.go +++ b/cmd/mesh-controller/retirement_test.go @@ -380,12 +380,12 @@ func TestNatsCleanupDeletesOnlyTheNamedRetiredConsumer(t *testing.T) { t.Fatalf("%+v", listing) } fake.deleted = nil - said = printed(t, func() error { return deleteFrom(t.Context(), conn, listing, 30, false, whyFlags(t, "tidy")) }) + said = printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, false, whyFlags(t, "tidy")) }) if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") || strings.Contains(said, "young") { t.Fatalf("deleted %v; said %s", fake.deleted, said) } - printed(t, func() error { return deleteFrom(t.Context(), conn, listing, 30, true, whyFlags(t, "tidy")) }) + printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, true, whyFlags(t, "tidy")) }) if !slices.Equal(fake.deleted, []string{"old"}) { t.Fatalf("confirmed, deleted %v", fake.deleted) } diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 9813068..a802c43 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -499,6 +499,15 @@ func (a *verbArguments) commandLine() ([]string, error) { return argv, nil } return []string{"cleanup", "list", "--json"}, nil + case "data": + argv := []string{"data", "--json"} + if m := str("machine"); m != "" { + argv = append(argv, "--machine", m) + } + if on("retired") { + argv = append(argv, "--retired") + } + return argv, nil case "doctor": which := 0 argv := []string{"doctor"} @@ -598,7 +607,7 @@ func (a *verbArguments) commandLine() ([]string, error) { // jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well. var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true, - "hand-acts": true, "durations": true, "conditions": true, "doctor": true, "retire": true, "cleanup": true} + "hand-acts": true, "durations": true, "conditions": true, "doctor": true, "retire": true, "cleanup": true, "data": true} // repairingCommand names a command line that repairs by hand, and so says why: a push, a plan stopped // or closed, a consumer re-made (novox/hq to-be 45 §7). Empty for any other. diff --git a/cmd/mesh-controller/seatverbs_schema_test.go b/cmd/mesh-controller/seatverbs_schema_test.go index 7363ebb..35656dc 100644 --- a/cmd/mesh-controller/seatverbs_schema_test.go +++ b/cmd/mesh-controller/seatverbs_schema_test.go @@ -276,6 +276,7 @@ var accountedFlags = map[string]map[string]string{ "conditions": {"json": "set by the verb: the answer is data"}, "retire": {"json": "set by the verb: the answer is data"}, "cleanup": {"json": "set by the verb: the answer is data"}, + "data": {"json": "set by the verb: the answer is data"}, "conditions history": {"json": "set by the verb: the answer is data"}, "conditions show": {"json": "set by the verb: the answer is data"}, "healers": {"json": "set by the verb: the answer is data"}, diff --git a/internal/broker/nats.go b/internal/broker/nats.go index b72b8d7..7b0cd29 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -133,7 +133,11 @@ type SeatVerb struct{ Seat, Verb string } // to the probe registry is the reason they cannot drift** — a probe that calls a verb its grant does // not name is refused by the bus on every run (found live on 2026-10-06: D8 timed out on each // machine, refused). Asked of any machine (`.*`), read-only verbs, nothing else of the seat. -var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}} +// +// D13 reads every machine's backup holder: what it measured of the data declared there (novox/hq ADR +// 0233). +var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}, + {Seat: "node-backup", Verb: "backed-up"}} // perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the // holder's machine (novox/hq ADR 0219): `paused.`, the build agent saying whether it takes work. diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 850937b..d8934f5 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -24,7 +24,7 @@ accounts { jetstream: enabled users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] } subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] } allow_responses: { max: 1, ttl: "1m" } } } diff --git a/internal/catalogue/backup_test.go b/internal/catalogue/backup_test.go index 0b46efc..7dc65f6 100644 --- a/internal/catalogue/backup_test.go +++ b/internal/catalogue/backup_test.go @@ -5,35 +5,10 @@ import ( "testing" ) -// A store provider says how its data is backed up (novox/hq ADR 0214); a provider of something that -// holds nothing does not have to. -func TestAStoreProviderWithoutABackupIsRefusedByTheCheck(t *testing.T) { - bare, err := ParseManifest([]byte(`{"module":"pg","version":"1", - "provides":[{"name":"postgres-database","scope":"mesh"}]}`)) - if err != nil { - t.Fatalf("parsing refused it, and the rule is the check's: %v", err) - } - if problems := CheckBackup(bare); len(problems) != 1 || !strings.Contains(problems[0], "node-backup") { - t.Fatalf("a store with no backup passed the check: %v", problems) - } - backed, err := ParseManifest([]byte(`{"module":"pg","version":"1", - "provides":[{"name":"postgres-database","scope":"mesh"}], - "resources":[{"id":"dumps","type":"directory","mode":"0700"}], - "contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:dumps}"}]}`)) - if err != nil { - t.Fatal(err) - } - if problems := CheckBackup(backed); len(problems) != 0 { - t.Fatalf("a store that contributes a backup was refused: %v", problems) - } - route, _ := ParseManifest([]byte(`{"module":"r","version":"1","provides":[{"name":"route","scope":"mesh"}]}`)) - if problems := CheckBackup(route); len(problems) != 0 { - t.Fatalf("a route was asked for a backup: %v", problems) - } -} - -// A backup contribution names its module's own directories; one it does not declare is refused -// where it is written rather than reaching the holder as the literal text. +// A backup contribution written by hand still names its module's own directories; one it does not +// declare is refused where it is written rather than reaching the holder as the literal text. (The +// catalogue check refuses a hand-written backup line at all since ADR 0233; parsing still reads one, +// because a module on the shelf was written before.) func TestABackupNamingAnUndeclaredDirectoryIsRefused(t *testing.T) { _, err := ParseManifest([]byte(`{"module":"pg","version":"1", "contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:nowhere}"}]}`)) @@ -42,9 +17,9 @@ func TestABackupNamingAnUndeclaredDirectoryIsRefused(t *testing.T) { } } -// The holder receives every module's backup lines with each module's own directories filled, each -// module's under a comment naming it — and a kind written in a shell's grammar is left untouched. -func TestBackupContributionsArePlacedWithTheirModulesDirectories(t *testing.T) { +// A module on the shelf from before ADR 0233, which writes its backup lines by hand and declares no +// data, is still backed up: its lines are placed as written, each module's under a comment naming it. +func TestHandWrittenBackupLinesOfAnOlderModuleArePlaced(t *testing.T) { pg, err := ParseManifest([]byte(`{"module":"pg","version":"1", "resources":[{"id":"dumps","type":"directory","path":"/srv/pg/dumps","mode":"0700"}], "contributions":[{"seat":"node-backup","kind":"backup","content":"run pg-dump-all\npath ${dir:dumps}"}]}`)) @@ -57,7 +32,7 @@ func TestBackupContributionsArePlacedWithTheirModulesDirectories(t *testing.T) { if err != nil { t.Fatal(err) } - placed, err := seatContributions([]Manifest{pg, mail}, BackupSeat, "backup", Rendering{}) + placed, err := seatContributions([]Manifest{pg, mail}, BackupSeat, "backup", Rendering{}, nil) if err != nil { t.Fatal(err) } diff --git a/internal/catalogue/catalogue_check_test.go b/internal/catalogue/catalogue_check_test.go index 998763b..5728a38 100644 --- a/internal/catalogue/catalogue_check_test.go +++ b/internal/catalogue/catalogue_check_test.go @@ -87,15 +87,18 @@ func TestNoCatalogueManifestNamesAnInstallation(t *testing.T) { } } -// TestEveryCatalogueStoreSaysHowItIsBackedUp is ADR 0214's check over the real catalogue: a module -// providing a store contributes a backup to node-backup, so a store added is a store backed up. -func TestEveryCatalogueStoreSaysHowItIsBackedUp(t *testing.T) { +// TestEveryCatalogueModuleDeclaresItsData is ADR 0233's check over the real catalogue (it replaced ADR +// 0214's store list): every provider that grants says what it keeps for its consumers, nothing writes a +// backup line by hand, every directory a container writes is declared, and every irreplaceable item is +// backed up — so a store added is a store backed up, without a list of stores to keep in step. +func TestEveryCatalogueModuleDeclaresItsData(t *testing.T) { root := catalogueRoot(t) found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) if err != nil || len(found) == 0 { t.Fatalf("no manifests under %s: %v", root, err) } - stores := 0 + shelf := Shelf{} + granting := 0 for _, p := range found { raw, err := os.ReadFile(p) if err != nil { @@ -105,18 +108,16 @@ func TestEveryCatalogueStoreSaysHowItIsBackedUp(t *testing.T) { if err != nil { continue // TestEveryCatalogueManifestParses says why } - for _, o := range m.Provides { - if storeProvisions[o.Name] { - stores++ - break - } - } - for _, problem := range CheckBackup(m) { - t.Error(problem) + if len(m.Grants) > 0 { + granting++ } + shelf[m.Module] = m } - if stores == 0 { - t.Fatal("no module in the catalogue provides a store, so this proved nothing") + for _, problem := range DataProblems(shelf) { + t.Error(problem) + } + if granting == 0 { + t.Fatal("no module in the catalogue grants a provision, so this proved nothing") } } diff --git a/internal/catalogue/data.go b/internal/catalogue/data.go new file mode 100644 index 0000000..26f2a8e --- /dev/null +++ b/internal/catalogue/data.go @@ -0,0 +1,766 @@ +package catalogue + +import ( + "bytes" + "encoding/json" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +// A module declares the data it holds, and the mesh protects and watches it from that declaration +// (novox/hq ADR 0233). +// +// **One section, `data`, for every kind of data a module keeps:** its own, by directory — a store's +// files, a mail spool, an application's uploads — or by an operator's path it was given; what it keeps +// for its consumers, by the provision they reach it through; and what of its own lives with a +// provider, by the provision it requires. Each entry has a class — how precious it is — and, for its +// own data, how it is protected; everything the mesh does is derived from those, never written per +// module: +// +// - a binding to a consumer's data does not move (ADR 0232) where the provision's class keeps data; +// - the backup holder's lines are composed from it — a module no longer writes them by hand; +// - what an unassignment leaves behind of an irreplaceable or valuable item is retired, listed by +// `cleanup list` and deleted only by `cleanup delete` (ADR 0230); +// - the self-check measures every item and says when one shrinks, disappears, stops being written, +// goes without its backup, sits on a degraded array, or is replaced by an empty copy of itself — +// urgent for what is irreplaceable, a warning for what is valuable. + +// The classes: how precious the data is. A fixed vocabulary, ranked by the operator (2026-10-06): a +// class is what the protections are derived from, so a new one is a decision, not a manifest's choice. +const ( + // ClassIrreplaceable is what must never be lost: the operator names it (the media library, the + // photo sites' storage). Protected by a backup or by a declared redundancy — one is required — + // retired rather than removed, and every alert about it is urgent. + ClassIrreplaceable = "irreplaceable" + // ClassValuable is anybody's work that would be painful to lose: in the nightly backup by default, + // retired rather than removed, and every alert about it a warning. + ClassValuable = "valuable" + // ClassRebuildable can be made again from something kept elsewhere — a clone, an index, a + // download, a night's dump — at a cost in time, not in data. In the nightly backup by default; + // forgotten when its module goes, and never alerted on. + ClassRebuildable = "rebuildable" + // ClassCache may be emptied at any moment with nothing lost but speed: never backed up, never + // measured, never alerted on. + ClassCache = "cache" + // ClassNone is a provision that keeps nothing of its consumers' (consumers only): a resolver, a + // certificate authority, an artifact store. + ClassNone = "none" +) + +// classRank orders the classes by how precious they are, so the stricter of two is chosen. +var classRank = map[string]int{ClassNone: 0, ClassCache: 1, ClassRebuildable: 2, ClassValuable: 3, ClassIrreplaceable: 4} + +// StricterClass is the more precious of two classes. +func StricterClass(a, b string) string { + if classRank[b] > classRank[a] { + return b + } + return a +} + +// Retires is whether a class's data is retired, not forgotten, when its machine no longer declares it. +func Retires(class string) bool { return class == ClassIrreplaceable || class == ClassValuable } + +// Watched is whether a class's data raises conditions: irreplaceable and valuable. +func Watched(class string) bool { return Retires(class) } + +// DefaultBackupWithin is how old the last good backup of an item may be before the self-check says so +// (novox/hq ADR 0214: a machine with data and no good backup in 48 hours). +const DefaultBackupWithin = 48 * time.Hour + +// Data is a manifest's `data` section. +type Data struct { + // Own is the data this module keeps itself. + Own []DataItem `json:"own,omitempty"` + // Consumers is what it keeps for its consumers, per provision it grants. + Consumers map[string]ConsumerData `json:"consumers,omitempty"` + // KeptBy is what of its own lives with the provider of a provision it requires — its rows, its + // objects — and how precious that is. The provider's protections follow the stricter of its own + // class for its consumers and this. + KeptBy map[string]KeptData `json:"kept-by,omitempty"` +} + +// DataItem is one piece of a module's own data. +type DataItem struct { + // ID names it within the module: what `data`, `cleanup` and a condition call it. + ID string `json:"id"` + // Path is one of the module's directories, `${dir:}`, or an operator's path it was given, + // `${access:}`, or a path beneath either. Never a machine path (novox/hq ADR 0112). + Path string `json:"path"` + Class string `json:"class"` + // Backup is how it is copied: "copy" (the holder reads it as it stands), "none", or a dump — a + // command that writes a consistent copy into another item, which is copied. Unsaid, anything but a + // cache is copied, unless it says it is protected by redundancy instead. + Backup *DataBackup `json:"backup,omitempty"` + // Redundancy says it is protected by the redundancy of the storage it lives on rather than by a + // copy, and why that is enough: the media library on an array there is no room to copy. The + // backup holder then watches that array, and a degraded one is said. + Redundancy string `json:"redundancy,omitempty"` + // Within is how old its last good backup may be; unsaid, DefaultBackupWithin. + Within string `json:"within,omitempty"` + // Active is how long it may go unwritten before that is a fault — for data something is + // expected to write all the time. Unsaid, a quiet item is not a fault. + Active string `json:"active,omitempty"` + // Why is a line for the reviewer: why this class. + Why string `json:"why,omitempty"` +} + +// ConsumerData is what a provider keeps for the consumers of one provision. +type ConsumerData struct { + Class string `json:"class"` + // In is where it lives: one of the module's own items (whose protection covers it), or a + // provision this module requires (whose provider keeps it, as its consumer). Unsaid only for none + // and cache. + In string `json:"in,omitempty"` + Why string `json:"why,omitempty"` +} + +// KeptData is how precious what a module keeps with a provider is. +type KeptData struct { + Class string `json:"class"` + Why string `json:"why,omitempty"` +} + +// DataBackup is how an item is copied. +type DataBackup struct { + Copy bool + None bool + // Dump is the command writing a consistent copy into the item Into. + Dump string + Into string +} + +// UnmarshalJSON reads "copy", "none" or {"dump": "...", "into": ""}. +func (b *DataBackup) UnmarshalJSON(raw []byte) error { + var word string + if err := json.Unmarshal(raw, &word); err == nil { + switch word { + case "copy": + *b = DataBackup{Copy: true} + case "none": + *b = DataBackup{None: true} + default: + return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}, not %q", word) + } + return nil + } + var full struct { + Dump string `json:"dump"` + Into string `json:"into"` + } + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&full); err != nil { + return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}: %w", err) + } + *b = DataBackup{Dump: full.Dump, Into: full.Into} + return nil +} + +// MarshalJSON writes it back in the form it was written. +func (b DataBackup) MarshalJSON() ([]byte, error) { + switch { + case b.Copy: + return json.Marshal("copy") + case b.None: + return json.Marshal("none") + } + return json.Marshal(struct { + Dump string `json:"dump"` + Into string `json:"into"` + }{b.Dump, b.Into}) +} + +// IsDump is whether the item is copied by a dump. +func (b *DataBackup) IsDump() bool { return b != nil && b.Dump != "" } + +// BackedUp is whether the holder keeps restore points of this item: anything but a cache by default — +// the nightly backup is the standard plan — unless it says "none", or says it is protected by +// redundancy and says nothing of a backup. +func (it DataItem) BackedUp() bool { + switch { + case it.Backup == nil: + return it.Class != ClassCache && it.Redundancy == "" + case it.Backup.None: + return false + } + return true +} + +// Protection is how the item is protected, in one word: "backup", "redundancy", both joined by "+", +// or "none". +func (it DataItem) Protection() string { + var by []string + if it.BackedUp() { + by = append(by, "backup") + } + if it.Redundancy != "" { + by = append(by, "redundancy") + } + if len(by) == 0 { + return "none" + } + return strings.Join(by, "+") +} + +// OwnedByModule is whether the item is in one of the module's own directories — the mesh's to retire — +// rather than an operator's path it was given, which the mesh never retires and never deletes. +func (it DataItem) OwnedByModule() bool { return strings.HasPrefix(it.Path, "${dir:") } + +// BackupWithin is the item's bound on its last good backup. +func (it DataItem) BackupWithin() time.Duration { + if d, err := ParseDataDuration(it.Within); err == nil && d > 0 { + return d + } + return DefaultBackupWithin +} + +// ActiveWithin is how long the item may go unwritten; zero when quiet is not a fault. +func (it DataItem) ActiveWithin() time.Duration { + d, _ := ParseDataDuration(it.Active) + return d +} + +// ParseDataDuration reads "48h", "90m" or "7d"; empty is zero. +func ParseDataDuration(s string) (time.Duration, error) { + s = strings.TrimSpace(s) + if s == "" { + return 0, nil + } + if days, ok := strings.CutSuffix(s, "d"); ok { + n, err := strconv.Atoi(days) + if err != nil || n <= 0 { + return 0, fmt.Errorf("%q is not a number of days", s) + } + return time.Duration(n) * 24 * time.Hour, nil + } + d, err := time.ParseDuration(s) + if err != nil || d <= 0 { + return 0, fmt.Errorf("%q is not a duration (48h, 90m, 7d)", s) + } + return d, nil +} + +// DataItems is the module's own data, in the order declared. +func (m Manifest) DataItems() []DataItem { + if m.Data == nil { + return nil + } + return m.Data.Own +} + +// DataItem is one own item by id. +func (m Manifest) DataItem(id string) (DataItem, bool) { + for _, it := range m.DataItems() { + if it.ID == id { + return it, true + } + } + return DataItem{}, false +} + +// ConsumerDataOf is what this module says it keeps for a provision's consumers, and whether it says. +func (m Manifest) ConsumerDataOf(provision string) (ConsumerData, bool) { + if m.Data == nil { + return ConsumerData{}, false + } + c, ok := m.Data.Consumers[provision] + return c, ok +} + +// KeptByOf is how precious what this module keeps with a provision's provider is, and whether it says. +func (m Manifest) KeptByOf(provision string) (KeptData, bool) { + if m.Data == nil { + return KeptData{}, false + } + k, ok := m.Data.KeptBy[provision] + return k, ok +} + +// keepsByClass is whether a class keeps something a binding must not move away from. +func keepsByClass(class string) bool { + return class == ClassIrreplaceable || class == ClassValuable || class == ClassRebuildable +} + +// dataID is what an item's id may be: it is a token in a condition's key and a word on a line. +var dataID = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + +// dataPathRef is an item's path: one of the module's directories or accesses, and optionally a path +// beneath it. +var dataPathRef = regexp.MustCompile(`^\$\{(dir|access):([a-z0-9][a-z0-9-]*)\}(/[^\s$]*)?$`) + +// dataProblems is what is wrong with the `data` section itself, from the manifest alone: judged at +// registration as every other per-manifest problem is. Whether a module declares what it should is +// the catalogue check's (DataProblems), because a module already running was written before it. +func (m Manifest) dataProblems() []string { + if m.Data == nil { + return nil + } + var problems []string + say := func(format string, args ...any) { + problems = append(problems, fmt.Sprintf("%s's data: ", m.Module)+fmt.Sprintf(format, args...)) + } + dirs := map[string]bool{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) == "directory" { + dirs[fmt.Sprint(r["id"])] = true + } + } + accesses := map[string]bool{} + for _, a := range m.Accesses { + if a.ID != "" { + accesses[a.ID] = true + } + } + ids := map[string]DataItem{} + paths := map[string]string{} + for i, it := range m.Data.Own { + label := it.ID + if label == "" { + label = fmt.Sprintf("item %d", i+1) + } + if !dataID.MatchString(it.ID) { + say("%s has no usable id: lower-case letters, digits and dashes", label) + } else if _, twice := ids[it.ID]; twice { + say("%s is declared twice", it.ID) + } + ids[it.ID] = it + ref := dataPathRef.FindStringSubmatch(it.Path) + switch { + case ref == nil: + say("%s's path %q is not one of the module's directories or accesses: ${dir:} or ${access:}, "+ + "or a path beneath one (a definition names no machine path, novox/hq ADR 0112)", label, it.Path) + case ref[1] == "dir" && !dirs[ref[2]]: + say("%s's path names ${dir:%s}, and %s declares no directory %q", label, ref[2], m.Module, ref[2]) + case ref[1] == "access" && !accesses[ref[2]]: + say("%s's path names ${access:%s}, and %s declares no access %q", label, ref[2], m.Module, ref[2]) + case strings.Contains(it.Path, ".."): + say("%s's path %q climbs out of its directory", label, it.Path) + } + if other, twice := paths[it.Path]; twice && it.Path != "" { + say("%s and %s name the same path %s", other, label, it.Path) + } + paths[it.Path] = label + switch it.Class { + case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache: + case ClassNone: + say("%s is class none, which only a provision keeping nothing of its consumers' is; own data "+ + "that is disposable is cache", label) + default: + say("%s's class %q is not one the mesh protects by: irreplaceable, valuable, rebuildable or cache", + label, it.Class) + } + if it.Class == ClassIrreplaceable && !it.BackedUp() && strings.TrimSpace(it.Redundancy) == "" { + say("%s is irreplaceable and is neither backed up nor said to be protected by redundancy; the only "+ + "copy of something is protected one way or the other (novox/hq ADR 0233) — copy it, dump it into "+ + "another item, or say `redundancy` with why that is enough", label) + } + if it.Class == ClassCache && it.Backup != nil && !it.Backup.None { + say("%s is a cache and asks to be backed up; a cache is disposable, or it is not a cache", label) + } + if it.Class == ClassCache && it.Redundancy != "" { + say("%s is a cache and says it is protected by redundancy; a cache is not protected", label) + } + if _, err := ParseDataDuration(it.Within); err != nil { + say("%s's within: %v", label, err) + } + if _, err := ParseDataDuration(it.Active); err != nil { + say("%s's active: %v", label, err) + } + if it.Within != "" && !it.BackedUp() { + say("%s states within, and is not backed up", label) + } + if it.Active != "" && !Watched(it.Class) { + say("%s is %s and expects writes; only irreplaceable and valuable data is watched", label, it.Class) + } + } + // Dumps go into an item of the same module, declared, and not into themselves. + for _, it := range m.Data.Own { + if it.Backup == nil || it.Backup.Copy || it.Backup.None { + continue + } + switch into, ok := ids[it.Backup.Into]; { + case strings.TrimSpace(it.Backup.Dump) == "": + say("%s's backup names no dump command", it.ID) + case it.Backup.Into == "": + say("%s's dump says no item it writes into", it.ID) + case !ok: + say("%s's dump writes into %q, which is not one of the module's data items", it.ID, it.Backup.Into) + case into.ID == it.ID: + say("%s's dump writes into itself; a dump is copied from where it lands", it.ID) + case into.Class == ClassCache: + say("%s's dump writes into %s, a cache; what is copied must not be disposable", it.ID, into.ID) + } + if it.Backup.Dump != "" { + declared := map[string]string{} + for d := range dirs { + declared[d] = "" + } + if _, err := dirFill(it.Backup.Dump, declared, m.Module); err != nil { + say("%s's dump: %v", it.ID, err) + } + } + } + provided := map[string]bool{} + for _, o := range m.Provides { + provided[o.Name] = true + } + wants := map[string]bool{} + for _, w := range m.Wants() { + wants[w] = true + } + for _, provision := range sortedKeys(m.Data.Consumers) { + c := m.Data.Consumers[provision] + if !provided[provision] { + say("says what it keeps for the consumers of %q, which it does not provide", provision) + } + switch c.Class { + case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache, ClassNone: + default: + say("its consumers' %s is class %q; one of irreplaceable, valuable, rebuildable, cache or none", provision, c.Class) + } + switch { + case c.Class == ClassNone && c.In != "": + say("its consumers' %s keeps nothing and says where it is kept (%s)", provision, c.In) + case keepsByClass(c.Class) && c.In == "": + say("its consumers' %s is %s and says nowhere it lives: `in` names one of the module's items, or a "+ + "provision it requires", provision, c.Class) + case c.In != "": + if own, ok := ids[c.In]; ok { + if c.Class == ClassIrreplaceable && !own.BackedUp() && own.Redundancy == "" { + say("its consumers' %s is irreplaceable and lives in %s, which is not protected", provision, c.In) + } + if classRank[own.Class] < classRank[c.Class] { + say("its consumers' %s is %s and lives in %s, which is only %s", provision, c.Class, c.In, own.Class) + } + } else if !wants[c.In] { + say("its consumers' %s lives in %q, which is neither one of its data items nor a provision it "+ + "requires", provision, c.In) + } + } + } + for _, provision := range sortedKeys(m.Data.KeptBy) { + k := m.Data.KeptBy[provision] + if !wants[provision] { + say("says it keeps data with the provider of %q, which it does not require", provision) + } + switch k.Class { + case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache: + default: + say("what it keeps with %s is class %q; one of irreplaceable, valuable, rebuildable or cache", provision, k.Class) + } + } + return problems +} + +// KeepsConsumerData is whether this module, providing a provision, keeps what each consumer writes +// there (novox/hq ADR 0232, ADR 0233): whether a consumer bound to it is bound to its data. +// +// **What the data section says, it gets**: irreplaceable, valuable or rebuildable keeps; cache and none +// do not — a cache lost in a move costs speed, not data. Before the section, an offer said it with +// `keeps-consumer-data`, which is still read; unsaid in both, a provider that grants each consumer a +// credential of its own keeps that consumer's data (ADR 0232 §1). The catalogue check refuses the +// unsaid case for a provider that grants (DataProblems), so the inference is what an older definition +// on the shelf gets, never a new one. +func (m Manifest) KeepsConsumerData(provision string) bool { + if c, ok := m.ConsumerDataOf(provision); ok { + return keepsByClass(c.Class) + } + for _, o := range m.Provides { + if o.Name == provision && o.KeepsConsumerData != nil { + return *o.KeepsConsumerData + } + } + _, grants := m.Grants[provision] + return grants +} + +// NeedsBackupHolder is whether a module's data needs the machine's backup holder to be there: it keeps +// something irreplaceable — backed up by the holder, or protected by an array the holder watches. A +// module keeping only valuable or rebuildable data is backed up where a holder is, and refused nowhere +// for want of one: the standard plan, not a requirement. +func (m Manifest) NeedsBackupHolder() bool { + for _, it := range m.DataItems() { + if it.Class == ClassIrreplaceable { + return true + } + } + return false +} + +// --- derived: the backup holder's lines --------------------------------------------------------- + +// The node-backup seat's kinds (novox/hq to-be 43, ADR 0233): `backup` is what to run and which paths +// to keep, `data` every item with its class and protection, for the holder to measure and watch. +const ( + BackupKindBackup = "backup" + BackupKindData = "data" +) + +// derivedContributions is what a module's data section gives the backup holder: its backup lines and +// its items. Every item is listed, so a valuable one on a machine is measured whether or not it is +// copied; a cache is listed and not measured. +func (m Manifest) derivedContributions() []SeatContribution { + items := m.DataItems() + if len(items) == 0 { + return nil + } + var lines []string + kept := map[string]bool{} + keep := func(path string) { + if !kept[path] { + kept[path] = true + lines = append(lines, "path "+path) + } + } + for _, it := range items { + if !it.BackedUp() { + continue + } + if it.Backup.IsDump() { + lines = append(lines, "run "+strings.TrimSpace(it.Backup.Dump)) + if into, ok := m.DataItem(it.Backup.Into); ok { + keep(into.Path) + } + continue + } + keep(it.Path) + } + var described []string + for _, it := range items { + covered := "-" + switch { + case it.Backup.IsDump(): + if into, ok := m.DataItem(it.Backup.Into); ok { + covered = into.Path + } + case it.BackedUp(): + covered = it.Path + } + described = append(described, fmt.Sprintf("item %s %s %s %s %s", it.ID, it.Class, it.Path, covered, it.Protection())) + } + var out []SeatContribution + if len(lines) > 0 { + out = append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindBackup, Content: strings.Join(lines, "\n") + "\n"}) + } + return append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindData, Content: strings.Join(described, "\n") + "\n"}) +} + +// allContributions is every contribution a module makes to a seat's holder: what it wrote, and what +// its data section derives. **One list**: a module that declares its data has its backup lines +// composed from it, and a backup line it still writes by hand is not placed — the catalogue check +// refuses it — so the holder never copies two lists that disagree. +func (m Manifest) allContributions() []SeatContribution { + if m.Data == nil { + return m.Contributions + } + derived := m.derivedContributions() + out := make([]SeatContribution, 0, len(m.Contributions)+len(derived)) + for _, c := range m.Contributions { + if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat { + continue + } + out = append(out, c) + } + return append(out, derived...) +} + +// --- the catalogue check ------------------------------------------------------------------------ + +// DataProblems is what the catalogue check refuses about the data a module declares (novox/hq ADR +// 0233), judged over the manifests given together: +// +// - a provider that grants a provision says what it keeps for that provision's consumers; +// - nobody says it on the offer any more (`keeps-consumer-data`): the data section is the one place; +// - nobody writes a backup line by hand: it is derived from the data section; +// - a directory a container writes, mounted whole, is a data item of some class; +// - consumer data said to live in a required provision lives where that provision's provider keeps +// its consumers' data, as preciously, when the provider is given; +// - data a consumer keeps with a provider as irreplaceable is protected there, when the provider is +// given. +// +// **The check's, not registration's**, as ADR 0214's backup rule was: a module already on the shelf +// was written before the rule, and refusing it there would refuse the very providers whose data the +// rule protects. Each module meets it where it is written. +func DataProblems(shelf Shelf) []string { + var problems []string + for _, name := range shelfOrder(shelf) { + m := shelf[name] + for _, provision := range sortedKeys(m.Grants) { + if _, said := m.ConsumerDataOf(provision); !said { + problems = append(problems, fmt.Sprintf( + "%s grants %s and does not say what it keeps for its consumers: data.consumers.%s with a "+ + "class — irreplaceable, valuable, rebuildable, cache, or none (novox/hq ADR 0233)", name, provision, provision)) + } + } + for _, o := range m.Provides { + if o.KeepsConsumerData != nil { + problems = append(problems, fmt.Sprintf( + "%s says keeps-consumer-data on its offer of %s; it is said once, in data.consumers.%s, with a "+ + "class (novox/hq ADR 0233)", name, o.Name, o.Name)) + } + } + for i, c := range m.Contributions { + if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat { + problems = append(problems, fmt.Sprintf( + "%s's contribution %d is a backup line written by hand; backups are derived from the data "+ + "section — declare the data, with its class and how it is protected (novox/hq ADR 0233)", name, i+1)) + } + } + for _, dir := range writtenDirectories(m) { + if !coversDirectory(m, dir) { + problems = append(problems, fmt.Sprintf( + "%s mounts its directory %q into a container to be written, and declares no data in it: "+ + "data.own with a class — cache if it is disposable (novox/hq ADR 0233)", name, dir)) + } + } + if m.Data == nil { + continue + } + for _, provision := range sortedKeys(m.Data.Consumers) { + c := m.Data.Consumers[provision] + if c.In == "" { + continue + } + if _, own := m.DataItem(c.In); own { + continue + } + for _, pname := range shelfOrder(shelf) { + p := shelf[pname] + pc, said := p.ConsumerDataOf(c.In) + if !said || !providesName(p, c.In) { + continue + } + if classRank[pc.Class] < classRank[c.Class] { + problems = append(problems, fmt.Sprintf( + "%s keeps its consumers' %s, %s, in %s — and %s keeps its consumers' %s as %s, "+ + "so it is not protected as %s", name, provision, c.Class, c.In, pname, c.In, pc.Class, c.Class)) + } + } + } + for _, provision := range sortedKeys(m.Data.KeptBy) { + k := m.Data.KeptBy[provision] + if k.Class != ClassIrreplaceable { + continue + } + for _, pname := range shelfOrder(shelf) { + p := shelf[pname] + pc, said := p.ConsumerDataOf(provision) + if !said || !providesName(p, provision) { + continue + } + if !keepsByClass(pc.Class) { + problems = append(problems, fmt.Sprintf( + "%s keeps irreplaceable data with %s, and %s keeps its consumers' %s as %s — nothing of it is "+ + "protected there", name, provision, pname, provision, pc.Class)) + continue + } + if in, own := p.DataItem(pc.In); own && !in.BackedUp() && in.Redundancy == "" { + problems = append(problems, fmt.Sprintf( + "%s keeps irreplaceable data with %s, and %s keeps it in %s, which is neither backed up nor "+ + "on declared redundancy", name, provision, pname, pc.In)) + } + } + } + } + return problems +} + +func providesName(m Manifest, provision string) bool { + for _, o := range m.Provides { + if o.Name == provision { + return true + } + } + return false +} + +// writtenDirectories are the module's directories a container mounts whole and may write: a volume +// `${dir:}:` without `:ro`, or a directory stated by an absolute path mounted the same +// way. A file beneath a directory, or a read-only mount, is configuration the mesh wrote. +func writtenDirectories(m Manifest) []string { + absolute := map[string]string{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "directory" { + continue + } + if p, ok := r["path"].(string); ok && strings.HasPrefix(p, "/") { + absolute[strings.TrimRight(p, "/")] = fmt.Sprint(r["id"]) + } + } + seen := map[string]bool{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "container" { + continue + } + vols, _ := r["volumes"].([]any) + for _, v := range vols { + s, _ := v.(string) + mount := volumeMount.FindStringSubmatch(s) + if mount == nil || volumeReadOnly(mount[3]) { + continue + } + src := strings.TrimRight(mount[1], "/") + if ref := dirPlain.FindStringSubmatch(src); ref != nil { + seen[ref[1]] = true + } else if id, ok := absolute[src]; ok { + seen[id] = true + } + } + } + out := make([]string, 0, len(seen)) + for id := range seen { + out = append(out, id) + } + sort.Strings(out) + return out +} + +// volumeMount is a container's volume, `:[:]`, its source possibly a +// `${dir:}` — whose own colon is not the separator. +var volumeMount = regexp.MustCompile(`^(\$\{(?:dir|access):[a-z0-9][a-z0-9-]*\}[^:]*|[^:]+):([^:]+)(?::(.*))?$`) + +// volumeReadOnly is whether a volume's options mount it read-only. +func volumeReadOnly(options string) bool { + for _, o := range strings.Split(options, ",") { + if strings.TrimSpace(o) == "ro" { + return true + } + } + return false +} + +// dirPlain is a whole directory, `${dir:}` and nothing after it. +var dirPlain = regexp.MustCompile(`^\$\{dir:([a-z0-9][a-z0-9-]*)\}$`) + +// coversDirectory is whether a data item names the directory, a path within it, or a directory it +// is placed beneath. +func coversDirectory(m Manifest, dir string) bool { + parents := map[string]string{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "directory" { + continue + } + if p, ok := r["path"].(string); ok { + if ref := dirRef.FindStringSubmatch(p); ref != nil && strings.HasPrefix(p, "${dir:") { + parents[fmt.Sprint(r["id"])] = ref[1] + } + } + } + for _, it := range m.DataItems() { + ref := dataPathRef.FindStringSubmatch(it.Path) + if ref == nil || ref[1] != "dir" { + continue + } + for d, hops := dir, 0; d != "" && hops < 4; d, hops = parents[d], hops+1 { + if ref[2] == d { + return true + } + } + } + return false +} diff --git a/internal/catalogue/data_test.go b/internal/catalogue/data_test.go new file mode 100644 index 0000000..e083f49 --- /dev/null +++ b/internal/catalogue/data_test.go @@ -0,0 +1,226 @@ +package catalogue + +import ( + "slices" + "strings" + "testing" +) + +// A store as a module now declares it (novox/hq ADR 0233): its own data by directory with a class and a +// dump, and what it keeps for its consumers. +const declaredStore = `{"module":"pg","version":"1", + "provides":[{"name":"postgres-database","scope":"mesh"}], + "grants":{"postgres-database":"${dir:grants}"}, + "data":{ + "own":[ + {"id":"store","path":"${dir:store}","class":"irreplaceable","backup":{"dump":"dump-all > ${dir:dumps}/all","into":"dumps"},"active":"1d"}, + {"id":"dumps","path":"${dir:dumps}","class":"rebuildable","backup":"none"}], + "consumers":{"postgres-database":{"class":"irreplaceable","in":"store"}}}, + "resources":[ + {"id":"grants","type":"directory","mode":"0700"}, + {"id":"store","type":"directory","path":"/srv/store","mode":"0700"}, + {"id":"dumps","type":"directory","path":"${dir:store}/dumps","mode":"0700"}, + {"id":"server","type":"container","image":"x@sha256:aa","volumes":["${dir:store}:/var/lib/postgresql/data"]}]}` + +func mustParse(t *testing.T, raw string) Manifest { + t.Helper() + m, err := ParseManifest([]byte(raw)) + if err != nil { + t.Fatalf("refused: %v", err) + } + return m +} + +func TestADeclaredStoreParsesAndPassesTheCheck(t *testing.T) { + m := mustParse(t, declaredStore) + if problems := DataProblems(Shelf{"pg": m}); len(problems) > 0 { + t.Fatalf("a store declaring its data was refused: %v", problems) + } + if !m.KeepsConsumerData("postgres-database") { + t.Fatal("an irreplaceable consumer class does not keep the consumer's data") + } + if it, _ := m.DataItem("store"); !it.BackedUp() || it.BackupWithin() != DefaultBackupWithin || it.ActiveWithin().Hours() != 24 { + t.Fatalf("the store item reads %+v", it) + } + if it, _ := m.DataItem("dumps"); it.BackedUp() { + t.Fatal("a rebuildable item that says none is backed up") + } +} + +// Every rule of the section itself, refused at parse in the words of the module. +func TestTheDataSectionIsRefusedWhereItIsWrong(t *testing.T) { + for _, c := range []struct{ name, data, want string }{ + {"a class it does not know", `{"own":[{"id":"a","path":"${dir:d}","class":"precious"}]}`, "not one the mesh protects by"}, + {"an access it does not have", `{"own":[{"id":"a","path":"${access:media}","class":"irreplaceable","redundancy":"an array"}]}`, "declares no access"}, + {"irreplaceable and protected by nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":"none"}]}`, "protected one way or the other"}, + {"a cache on redundancy", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","redundancy":"an array"}]}`, "a cache is not protected"}, + {"kept with something not required", `{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}`, "does not require"}, + {"a cache expecting writes", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","active":"1d"}]}`, "only irreplaceable and valuable"}, + {"a machine path", `{"own":[{"id":"a","path":"/srv/a","class":"cache"}]}`, "names no machine path"}, + {"a directory it does not declare", `{"own":[{"id":"a","path":"${dir:nowhere}","class":"cache"}]}`, "declares no directory"}, + {"a cache backed up", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","backup":"copy"}]}`, "disposable"}, + {"a dump into nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":{"dump":"x","into":"b"}}]}`, "not one of the module's data items"}, + {"a dump into itself", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":{"dump":"x","into":"a"}}]}`, "into itself"}, + {"a bad duration", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","within":"soon"}]}`, "not a duration"}, + {"the same id twice", `{"own":[{"id":"a","path":"${dir:d}","class":"cache"},{"id":"a","path":"${dir:d}/x","class":"cache"}]}`, "declared twice"}, + {"own data of class none", `{"own":[{"id":"a","path":"${dir:d}","class":"none"}]}`, "only a provision"}, + {"consumers of something not provided", `{"consumers":{"s3-bucket":{"class":"cache"}}}`, "does not provide"}, + {"consumer data that lives nowhere", `{"consumers":{"q":{"class":"irreplaceable"}}}`, "says nowhere it lives"}, + {"consumer data in an item less precious", `{"own":[{"id":"a","path":"${dir:d}","class":"rebuildable"}],"consumers":{"q":{"class":"irreplaceable","in":"a"}}}`, "which is only rebuildable"}, + {"consumer data in something unknown", `{"consumers":{"q":{"class":"rebuildable","in":"elsewhere"}}}`, "neither one of its data items"}, + } { + raw := `{"module":"m","version":"1","provides":["q"],"resources":[{"id":"d","type":"directory","mode":"0700"}],"data":` + c.data + `}` + _, err := ParseManifest([]byte(raw)) + if err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: %v, want %q", c.name, err, c.want) + } + } +} + +// The catalogue check: a granting provider says what it keeps, says it in one place, writes no backup +// line by hand, and every directory a container writes is declared (novox/hq ADR 0233). +func TestTheCatalogueCheckAsksEveryModuleForItsData(t *testing.T) { + unsaid := mustParse(t, `{"module":"q","version":"1","provides":[{"name":"queue","scope":"mesh"}], + "grants":{"queue":"${dir:g}"},"resources":[{"id":"g","type":"directory","mode":"0700"}]}`) + onTheOffer := mustParse(t, `{"module":"r","version":"1","provides":[{"name":"resolver","scope":"mesh","keeps-consumer-data":false}]}`) + byHand := mustParse(t, `{"module":"h","version":"1","resources":[{"id":"d","type":"directory","mode":"0700"}], + "contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:d}"}]}`) + writes := mustParse(t, `{"module":"w","version":"1","resources":[{"id":"d","type":"directory","mode":"0700"}, + {"id":"c","type":"directory","mode":"0700"}, + {"id":"app","type":"container","image":"x@sha256:aa","volumes":["${dir:d}:/data","${dir:c}:/etc/app:ro","${dir:d}/f:/f"]}]}`) + problems := strings.Join(DataProblems(Shelf{"q": unsaid, "r": onTheOffer, "h": byHand, "w": writes}), "\n") + for _, want := range []string{ + "q grants queue and does not say what it keeps", + "r says keeps-consumer-data on its offer", + "h's contribution 1 is a backup line written by hand", + `w mounts its directory "d" into a container to be written`, + } { + if !strings.Contains(problems, want) { + t.Errorf("the check does not say %q:\n%s", want, problems) + } + } + if strings.Contains(problems, `"c"`) { + t.Errorf("a read-only mount was taken for written data:\n%s", problems) + } + // The same module declaring the directory, as a cache, passes. + declared := mustParse(t, `{"module":"w","version":"1","data":{"own":[{"id":"d","path":"${dir:d}","class":"cache"}]}, + "resources":[{"id":"d","type":"directory","mode":"0700"}, + {"id":"app","type":"container","image":"x@sha256:aa","volumes":["${dir:d}:/data"]}]}`) + if p := DataProblems(Shelf{"w": declared}); len(p) > 0 { + t.Fatalf("a declared directory is still refused: %v", p) + } +} + +// Consumer data said to live in a provision the module requires is protected only as well as that +// provision's provider protects its consumers' data. +func TestConsumerDataInARequiredProvisionIsJudgedAgainstItsProvider(t *testing.T) { + idp := mustParse(t, `{"module":"idp","version":"1","requires":["postgres-database"], + "provides":[{"name":"oidc-client","scope":"mesh"}],"grants":{"oidc-client":"${dir:g}"}, + "resources":[{"id":"g","type":"directory","mode":"0700"}], + "data":{"consumers":{"oidc-client":{"class":"irreplaceable","in":"postgres-database"}}}}`) + cachy := mustParse(t, strings.Replace(declaredStore, `"consumers":{"postgres-database":{"class":"irreplaceable","in":"store"}}`, + `"consumers":{"postgres-database":{"class":"cache"}}`, 1)) + if p := strings.Join(DataProblems(Shelf{"idp": idp, "pg": cachy}), "\n"); !strings.Contains(p, "not protected as irreplaceable") { + t.Fatalf("irreplaceable data kept in a cache passed: %s", p) + } + if p := DataProblems(Shelf{"idp": idp, "pg": mustParse(t, declaredStore)}); len(p) > 0 { + t.Fatalf("refused against a provider that keeps its consumers' data: %v", p) + } +} + +// What a binding is sticky by follows the class (ADR 0232, generalised by 0233): a cache's consumers +// move freely, a store's do not; an older definition saying nothing still follows its grant. +func TestKeepingConsumerDataFollowsTheClass(t *testing.T) { + for class, keeps := range map[string]bool{"irreplaceable": true, "valuable": true, "rebuildable": true, "cache": false, "none": false} { + in := `,"in":"d"` + own := `"own":[{"id":"d","path":"${dir:d}","class":"irreplaceable"}],` + if !keeps { + in, own = "", "" + } + m := mustParse(t, `{"module":"p","version":"1","provides":[{"name":"q","scope":"mesh"}],"grants":{"q":"${dir:d}"}, + "resources":[{"id":"d","type":"directory","mode":"0700"}], + "data":{`+own+`"consumers":{"q":{"class":"`+class+`"`+in+`}}}}`) + if m.KeepsConsumerData("q") != keeps { + t.Errorf("class %s keeps consumer data: %v, want %v", class, !keeps, keeps) + } + shelf := map[string]Manifest{"p": m} + if KeepsConsumerData(shelf, "q") != keeps { + t.Errorf("class %s: the provision by name keeps: %v", class, !keeps) + } + } + older := mustParse(t, `{"module":"p","version":"1","provides":[{"name":"q","scope":"mesh"}],"grants":{"q":"${dir:d}"}, + "resources":[{"id":"d","type":"directory","mode":"0700"}]}`) + if !older.KeepsConsumerData("q") { + t.Fatal("an older definition that grants no longer keeps its consumers' data") + } +} + +// The backup holder's lines are derived: the dump runs and the directory it writes into is kept, +// anything but a cache is copied by default — the standard nightly plan — an item on redundancy is not +// copied, and every item is listed with its class and protection for the holder to measure and watch: +// directories filled, a home directory filled from the machine, an operator's path from where the +// assignment placed it. A backup line still written by hand beside a data section is not placed. +func TestTheBackupHoldersLinesAreDerivedFromTheData(t *testing.T) { + pg := mustParse(t, declaredStore) + pg.Contributions = append(pg.Contributions, SeatContribution{Seat: BackupSeat, Kind: "backup", Content: "path /stale"}) + agent := mustParse(t, `{"module":"agent","version":"1", + "data":{"own":[{"id":"home","path":"${dir:home}","class":"valuable"},{"id":"tmp","path":"${dir:home}/tmp","class":"cache"}]}, + "resources":[{"id":"home","type":"directory","path":"${machine:account-home}/.agent","mode":"0700"}]}`) + media := mustParse(t, `{"module":"media","version":"1", + "accesses":[{"id":"films","mode":"read"}], + "data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"on a redundant array; no room for a copy"}, + {"id":"meta","path":"${dir:meta}","class":"rebuildable"}]}, + "resources":[{"id":"meta","type":"directory","mode":"0700"}]}`) + facts := map[string]string{"account-home": "/home/op"} + with := Rendering{Settings: SettingsBy{"media": {{From: "node", Values: map[string]any{AccessesSetting: map[string]any{"films": "/tank/films"}}}}}} + backup, err := seatContributions([]Manifest{pg, agent, media}, BackupSeat, BackupKindBackup, with, facts) + if err != nil { + t.Fatal(err) + } + want := "# agent\npath /home/op/.agent\n# media\npath /var/lib/media/meta\n# pg\nrun dump-all > /srv/store/dumps/all\npath /srv/store/dumps\n" + if backup != want { + t.Fatalf("backup lines:\n%s\nwant:\n%s", backup, want) + } + data, err := seatContributions([]Manifest{pg, agent, media}, BackupSeat, BackupKindData, with, facts) + if err != nil { + t.Fatal(err) + } + want = "# agent\nitem home valuable /home/op/.agent /home/op/.agent backup\nitem tmp cache /home/op/.agent/tmp - none\n" + + "# media\nitem films irreplaceable /tank/films - redundancy\nitem meta rebuildable /var/lib/media/meta /var/lib/media/meta backup\n" + + "# pg\nitem store irreplaceable /srv/store /srv/store/dumps backup\nitem dumps rebuildable /srv/store/dumps - none\n" + if data != want { + t.Fatalf("data lines:\n%s\nwant:\n%s", data, want) + } + if _, err := seatContributions([]Manifest{agent}, BackupSeat, BackupKindData, Rendering{}, nil); err == nil { + t.Fatal("a home directory with no account on the machine reached the holder as a placeholder") + } + // What keeps something irreplaceable depends on the machine's backup holder — it backs it up or + // watches its array; valuable data alone is backed up where a holder is, and refused nowhere. + if !slices.Contains(DependsOn(media), BackupSeat) || !slices.Contains(DependsOn(pg), BackupSeat) || + slices.Contains(DependsOn(agent), BackupSeat) { + t.Fatalf("depends: media %v, pg %v, agent %v", DependsOn(media), DependsOn(pg), DependsOn(agent)) + } + if it, _ := media.DataItem("films"); it.OwnedByModule() || it.Protection() != "redundancy" { + t.Fatalf("an operator's path reads %+v", it) + } +} + +// Data a consumer keeps with a provider as irreplaceable must be protected there: the check refuses a +// provider that keeps its consumers' data as a cache, or in an item with no protection. +func TestIrreplaceableDataKeptWithAProviderMustBeProtectedThere(t *testing.T) { + photos := mustParse(t, `{"module":"photos","version":"1","requires":["s3-bucket"], + "data":{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}}`) + store := func(backup string) Manifest { + return mustParse(t, `{"module":"objects","version":"1","provides":[{"name":"s3-bucket","scope":"mesh"}], + "grants":{"s3-bucket":"${dir:g}"}, + "data":{"own":[{"id":"data","path":"${dir:data}","class":"valuable","backup":"`+backup+`"}], + "consumers":{"s3-bucket":{"class":"valuable","in":"data"}}}, + "resources":[{"id":"g","type":"directory","mode":"0700"},{"id":"data","type":"directory","mode":"0700"}]}`) + } + if p := DataProblems(Shelf{"photos": photos, "objects": store("copy")}); len(p) > 0 { + t.Fatalf("a provider backing its consumers' data up was refused: %v", p) + } + if p := strings.Join(DataProblems(Shelf{"photos": photos, "objects": store("none")}), "\n"); !strings.Contains(p, "neither backed up") { + t.Fatalf("irreplaceable photos in an unprotected store passed: %s", p) + } +} diff --git a/internal/catalogue/environment_into.go b/internal/catalogue/environment_into.go index 52f662e..2cb6c17 100644 --- a/internal/catalogue/environment_into.go +++ b/internal/catalogue/environment_into.go @@ -576,7 +576,7 @@ func contributionsInto(resource map[string]any, m Manifest, modules []Manifest, found := ofContributed.FindStringSubmatch(placeholder) first, second, _ := strings.Cut(found[2], ":") if found[1] == "contribution" { - placed, err := seatContributions(modules, first, second, with) + placed, err := seatContributions(modules, first, second, with, facts) if err != nil && failed == nil { failed = err } diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 394dd4d..b2ca58c 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -239,25 +239,6 @@ func (m Manifest) IdentityBoundOf(provision string) IdentityBound { return IdentityBound{} } -// KeepsConsumerData is whether this module, providing a provision, keeps what each consumer writes -// there (novox/hq ADR 0232): whether a consumer bound to it is bound to its data. -// -// **What an offer says, it gets.** Unsaid, it follows from whether the provider grants each consumer -// a credential of its own: a provider that does makes an account for every consumer — a role and its -// database, a key and its bucket, a client — and what the consumer writes under that account stays -// with that provider. One that grants nothing keeps nothing of anybody's: the resolver, a CA, the -// artifact store each answer any consumer alike, and moving a consumer between two of them loses -// nothing. -func (m Manifest) KeepsConsumerData(provision string) bool { - for _, o := range m.Provides { - if o.Name == provision && o.KeepsConsumerData != nil { - return *o.KeepsConsumerData - } - } - _, grants := m.Grants[provision] - return grants -} - // KeepsConsumerData is whether a provision, by name, keeps its consumers' data across the // catalogue: true when any module providing it at the mesh's scope does. **A property of the name**, // as brokering is: two providers disagreeing would make the same binding sticky or free depending on @@ -466,6 +447,12 @@ type Manifest struct { // (novox/hq ADR 0201). Not history — that is an event — and never a secret, sealed or not. State []StateDeclaration `json:"state,omitempty"` + // Data is every kind of data this module keeps — its own, by directory, and what it keeps for + // its consumers, by provision — each with a class the mesh protects and watches it by (novox/hq + // ADR 0233). One list: the backup holder's lines, the bindings that do not move, what an + // unassignment retires and what the self-check measures are all derived from it. + Data *Data `json:"data,omitempty"` + // Reads are other modules' state this module reads and watches, each `.` // (novox/hq ADR 0201). Read-only: only the owner's instances write. Reads []string `json:"reads,omitempty"` @@ -2007,6 +1994,7 @@ func ParseManifest(raw []byte) (Manifest, error) { problems = append(problems, m.shellProblems()...) problems = append(problems, m.contributionPlaceholderProblems()...) problems = append(problems, m.seatContributionProblems()...) + problems = append(problems, m.dataProblems()...) for i, r := range m.Resources { id, _ := r["id"].(string) diff --git a/internal/catalogue/seat_contributions.go b/internal/catalogue/seat_contributions.go index fbd51b1..1236512 100644 --- a/internal/catalogue/seat_contributions.go +++ b/internal/catalogue/seat_contributions.go @@ -23,46 +23,6 @@ const MessageBusSeat = "node-message-bus" // BackupSeat is the machine's backups (novox/hq ADR 0214, to-be 43). const BackupSeat = "node-backup" -// storeProvisions are the provisions whose provider keeps its consumers' data (novox/hq ADR 0214): -// a module providing one must say how to back it up, or the data the mesh hands out is the data it -// cannot restore — issue 241's seven databases. A provision that holds nothing worth keeping (a -// route, a cache, a name) is not here; adding one is adding a store. -var storeProvisions = map[string]bool{ - "postgres-database": true, - "mssql-database": true, - "mongodb-database": true, - "s3-bucket": true, - "influxdb-api": true, - "secret": true, -} - -// CheckBackup is a store provider that contributes no backup (novox/hq ADR 0214, "How it is -// checked"). -// -// **The catalogue check's, not parsing's.** A manifest already registered and running was written -// before the rule; refusing it on read would make the controller refuse the very providers whose -// data the rule protects. New definitions meet it in the catalogue check, where they are written. -func CheckBackup(m Manifest) []string { - backs := false - for _, c := range m.Contributions { - if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat && c.Kind == "backup" { - backs = true - } - } - if backs { - return nil - } - var problems []string - for _, o := range m.Provides { - if storeProvisions[o.Name] { - problems = append(problems, fmt.Sprintf( - "%s provides %s, which keeps its consumers' data, and contributes no backup to %s; a "+ - "store says how its data is copied (novox/hq ADR 0214)", m.Module, o.Name, BackupSeat)) - } - } - return problems -} - // SeatContribution is one piece of configuration a module gives a seat's holder to place. type SeatContribution struct { // Seat is the seat whose holder places it. @@ -188,7 +148,11 @@ func seatPlaceholderProblems(m Manifest, r map[string]any) []string { // comment line naming it in the tool's grammar, and empty when nothing is contributed. A kind that // takes directories has each contributor's `${dir:}` filled with where that module's directory // is on this machine (novox/hq to-be 43). -func seatContributions(modules []Manifest, seat, kind string, with Rendering) (string, error) { +// +// A directory placed in a person's home is named `${machine:account-home}/…` (novox/hq to-be 29); in a +// kind that takes directories that is filled from the machine's facts as well, so the holder reads a +// path, never a placeholder (novox/hq ADR 0233: an item in the operator's home is backed up too). +func seatContributions(modules []Manifest, seat, kind string, with Rendering, facts map[string]string) (string, error) { s, r, ok := receivable(seat, kind) if !ok { return "", nil @@ -197,7 +161,7 @@ func seatContributions(modules []Manifest, seat, kind string, with Rendering) (s var b strings.Builder for _, m := range inModuleOrder(modules) { named := false - for _, c := range m.Contributions { + for _, c := range m.allContributions() { if c.Kind != kind { continue } @@ -214,6 +178,27 @@ func seatContributions(modules []Manifest, seat, kind string, with Rendering) (s if err != nil && failed == nil { failed = err } + // An operator's path the module was given, as an item of data on it (novox/hq ADR 0233). + if accessRef.MatchString(filled) { + _, byID, err := accessesFor(m, with.Settings[m.Module]) + if err == nil { + filled, err = accessFill(filled, byID, m.Module) + } + if err != nil && failed == nil { + failed = err + } + } + for _, key := range machineUsed(filled) { + value, has := facts[key] + if !has { + if failed == nil { + failed = fmt.Errorf("%s's %s for %s says ${machine:%s}, and this machine says %s", + m.Module, kind, s.Name, key, orNothing(namesOfFacts(facts))) + } + continue + } + filled = strings.ReplaceAll(filled, "${machine:"+key+"}", value) + } content = filled } b.WriteString(content) diff --git a/internal/catalogue/seat_dependencies.go b/internal/catalogue/seat_dependencies.go index d9818ce..dc2e0f0 100644 --- a/internal/catalogue/seat_dependencies.go +++ b/internal/catalogue/seat_dependencies.go @@ -123,6 +123,12 @@ func contributedTo(m Manifest) []string { out = append(out, s.Name) } } + // And the backup holder for what is irreplaceable (novox/hq ADR 0233): it backs it up or watches the + // array it is on. What is valuable is backed up where a holder is — the standard plan — and makes + // no machine refuse it for want of one. + if m.NeedsBackupHolder() { + out = append(out, BackupSeat) + } return out } diff --git a/internal/catalogue/seats.go b/internal/catalogue/seats.go index 5bef725..4ebd6b8 100644 --- a/internal/catalogue/seats.go +++ b/internal/catalogue/seats.go @@ -256,8 +256,10 @@ var defaultSeats = append([]Seat{ // disasters. A module contributes `backup` lines — what to run to take a consistent copy, and // which of its directories to keep. {Name: BackupSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0214", - Serves: backupVerbs(), - Receives: []Receivable{{Kind: "backup", Comment: "#", Dirs: true}}}, + Serves: backupVerbs(), + // `backup` is what to run and which paths to keep; `data` every item a module declares, with its + // class, for the holder to measure (novox/hq ADR 0233). Both derived from modules' data sections. + Receives: []Receivable{{Kind: BackupKindBackup, Comment: "#", Dirs: true}, {Kind: BackupKindData, Comment: "#", Dirs: true}}}, // Deferred (novox/hq ADR 0121): renaming to mesh-private-network is a scope + server/client // model change, not a rename, so it stays until that is built. {Name: "the-private-network", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index c54d7c4..33d5f43 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -282,8 +282,10 @@ var ControllerVerbs = []Verb{ "cause": "with answer: the cause in a word (retire-waiting when absent)", }, nil)}, {Name: "cleanup", Description: "Every consumer a provider holds retired — its age, its size where the " + - "backend knows, and why it was retired. With consumer (and node, module): the provider deletes that one " + - "retired consumer — never an active one. With older-than: every retired consumer older than that many " + + "backend knows, and why it was retired — and every module's own data retired on its machine (ADR 0233). " + + "With consumer (and node, module): the provider deletes that one retired consumer — never an active one; " + + "for a module's own retired item, consumer names the item and the machine's backup holder takes a last " + + "restore point of it, then deletes it. With older-than: every retired consumer older than that many " + "days, listed; deleted only with confirm. Deleting is a hand act, which says why (novox/hq ADR 0230).", Input: schema(map[string]string{ "node": "with consumer: the machine the provider runs on", @@ -294,6 +296,15 @@ var ControllerVerbs = []Verb{ "why": "with consumer or older-than: why — required, and recorded in the hand-act log", "cause": "with consumer or older-than: the cause in a word (cleanup-waiting when absent)", }, nil, "confirm")}, + // The data every machine declares (novox/hq ADR 0233). + {Name: "data", Description: "Every item of data every machine declares, as the self-check last measured it: " + + "its class (irreplaceable, rebuildable, cache), where it is, its size, its newest write, its newest good " + + "backup and the bound on it — and what is retired: kept after its module left the machine, removed only by " + + "`cleanup delete` (novox/hq ADR 0233).", + Input: schema(map[string]string{ + "machine": "one machine (optional)", + "retired": "\"true\": only what is retired", + }, nil, "retired")}, {Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " + "`builds` with that id follows it line by line, and the module is registered when the outcome comes.", Input: schema(map[string]string{ diff --git a/internal/inventory/data.go b/internal/inventory/data.go new file mode 100644 index 0000000..f7a7645 --- /dev/null +++ b/internal/inventory/data.go @@ -0,0 +1,294 @@ +package inventory + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// The data a module declares, as the mesh found it on each machine (novox/hq ADR 0233). +// +// The self-check composes what every machine declares and asks its backup holder what it measured; +// this keeps both. An irreplaceable item a machine no longer declares — its module unassigned, or no +// longer pulled in — is retired here, not forgotten: kept, with when and why, until a person deletes +// it through `cleanup delete`. + +// DeclaredData is one item a machine's composition declares now. +type DeclaredData struct { + Module, Item, Class string + // Owned is whether it is in the module's own directory: only that is the mesh's to retire. + Owned bool + // Protection is backup, redundancy, both ("backup+redundancy") or none. + Protection string +} + +// Redundancy is the redundant storage an item is on, as its machine's holder read it. +type Redundancy struct { + Kind string `json:"kind"` + Where string `json:"where"` + Healthy *bool `json:"healthy"` + Said string `json:"said"` +} + +// Measurement is what a machine's backup holder said of one item. Nil fields were not measured. +type Measurement struct { + Path string + Size *int64 + LastWrite *time.Time + MeasuredAt *time.Time + LastBackup *time.Time + // Error is what the holder could not measure, when it could not. + Error string + Redundancy *Redundancy +} + +// DataRecord is one item as the mesh keeps it. +type DataRecord struct { + Machine, Module, Item, Class, Path string + Owned bool + Protection string + FirstSeen, DeclaredAt time.Time + MeasureError string + Redundancy *Redundancy + Size *int64 + LastWrite, MeasuredAt, LastBackup *time.Time + RetiredAt *time.Time + RetiredWhy string + DeletedAt *time.Time + DeletedBy, DeletedWhy string +} + +// Retired is whether the item is retired and not deleted. +func (r DataRecord) Retired() bool { return r.RetiredAt != nil && r.DeletedAt == nil } + +// Key names it in one string, the way conditions and verbs do: //. +func (r DataRecord) Key() string { return r.Machine + "/" + r.Module + "/" + r.Item } + +// DataChange is what recording a machine's data changed: what it retired and what came back. +type DataChange struct { + Retired, Reenabled []DataRecord +} + +// readingEvery is how often a measurement is kept as a reading: the shrink is read over days, and a +// row every five minutes would say the same thing sixty times an hour. +const readingEvery = 55 * time.Minute + +// readingsKept is how long readings are kept. +const readingsKept = 90 * 24 * time.Hour + +// dataKey is one item's identity on one machine. +type dataKey struct{ module, item string } + +// RecordData keeps what one machine declares now and what its holder measured, at now. +// +// **Only from a composition that worked.** The caller passes the declared set of a machine whose +// composition succeeded; an item missing from it is then really no longer declared. An irreplaceable +// one is retired — never deleted, never forgotten — and anything else is forgotten, because what is +// rebuildable or a cache is not the mesh's to keep track of once its module is gone. An item declared +// again comes back from retirement as it was. +func (i *Inventory) RecordData(ctx context.Context, machine string, declared []DeclaredData, + measured map[string]map[string]Measurement, why string, now time.Time) (DataChange, error) { + var change DataChange + tx, err := i.store.Pool().Begin(ctx) + if err != nil { + return change, err + } + defer tx.Rollback(ctx) //nolint:errcheck + + existing := map[dataKey]DataRecord{} + rows, err := tx.Query(ctx, dataSelect+` where machine = $1`, machine) + if err != nil { + return change, err + } + for rows.Next() { + r, err := scanData(rows) + if err != nil { + rows.Close() + return change, err + } + existing[dataKey{r.Module, r.Item}] = r + } + rows.Close() + if err := rows.Err(); err != nil { + return change, err + } + + seen := map[dataKey]bool{} + for _, d := range declared { + k := dataKey{d.Module, d.Item} + seen[k] = true + m := measured[d.Module][d.Item] + var red struct { + Kind, Where, Said *string + Healthy *bool + } + if r := m.Redundancy; r != nil { + red.Kind, red.Where, red.Said, red.Healthy = &r.Kind, &r.Where, &r.Said, r.Healthy + } + was, had := existing[k] + if had && was.Retired() { + change.Reenabled = append(change.Reenabled, was) + } + // Deleted and declared again is new data: it starts over. + fresh := !had || was.DeletedAt != nil + if _, err := tx.Exec(ctx, ` + insert into data_item (machine, module, item, class, path, first_seen, declared_at, + size_bytes, last_write, measured_at, last_backup, owned, protection, + measure_error, redundancy_kind, redundancy_where, redundancy_healthy, redundancy_said) + values ($1, $2, $3, $4, $5, $6, $6, $7, $8, $9, $10, $12, $13, $14, $15, $16, $17, $18) + on conflict (machine, module, item) do update set + class = excluded.class, + owned = excluded.owned, + protection = excluded.protection, + measure_error = case when excluded.measured_at is not null then excluded.measure_error else data_item.measure_error end, + redundancy_kind = case when excluded.measured_at is not null then excluded.redundancy_kind else data_item.redundancy_kind end, + redundancy_where = case when excluded.measured_at is not null then excluded.redundancy_where else data_item.redundancy_where end, + redundancy_healthy = case when excluded.measured_at is not null then excluded.redundancy_healthy else data_item.redundancy_healthy end, + redundancy_said = case when excluded.measured_at is not null then excluded.redundancy_said else data_item.redundancy_said end, + path = case when excluded.path <> '' then excluded.path else data_item.path end, + first_seen = case when $11::boolean then excluded.first_seen else data_item.first_seen end, + declared_at = excluded.declared_at, + size_bytes = coalesce(excluded.size_bytes, data_item.size_bytes), + last_write = coalesce(excluded.last_write, data_item.last_write), + measured_at = coalesce(excluded.measured_at, data_item.measured_at), + last_backup = coalesce(excluded.last_backup, data_item.last_backup), + retired_at = null, retired_why = null, + deleted_at = null, deleted_by = null, deleted_why = null`, + machine, d.Module, d.Item, d.Class, m.Path, now, m.Size, m.LastWrite, m.MeasuredAt, m.LastBackup, + fresh, d.Owned, d.Protection, nullable(m.Error), red.Kind, red.Where, red.Healthy, red.Said); err != nil { + return change, err + } + if m.Size != nil && m.MeasuredAt != nil { + if _, err := tx.Exec(ctx, ` + insert into data_reading (machine, module, item, at, size_bytes, last_write) + select $1, $2, $3, $4, $5, $6 + where not exists (select 1 from data_reading + where machine = $1 and module = $2 and item = $3 and at > $4::timestamptz - $7::interval)`, + machine, d.Module, d.Item, *m.MeasuredAt, *m.Size, m.LastWrite, + fmt.Sprintf("%d seconds", int(readingEvery.Seconds()))); err != nil { + return change, err + } + } + } + for k, r := range existing { + if seen[k] || r.DeletedAt != nil || r.Retired() { + continue + } + if !catalogue.Retires(r.Class) || !r.Owned { + if _, err := tx.Exec(ctx, `delete from data_item where machine = $1 and module = $2 and item = $3`, + machine, k.module, k.item); err != nil { + return change, err + } + continue + } + if _, err := tx.Exec(ctx, `update data_item set retired_at = $4, retired_why = $5 + where machine = $1 and module = $2 and item = $3`, machine, k.module, k.item, now, why); err != nil { + return change, err + } + at := now + r.RetiredAt, r.RetiredWhy = &at, why + change.Retired = append(change.Retired, r) + } + if _, err := tx.Exec(ctx, `delete from data_reading where at < $1`, now.Add(-readingsKept)); err != nil { + return change, err + } + return change, tx.Commit(ctx) +} + +const dataSelect = `select machine, module, item, class, path, first_seen, declared_at, size_bytes, last_write, + measured_at, last_backup, retired_at, coalesce(retired_why, ''), deleted_at, coalesce(deleted_by, ''), + coalesce(deleted_why, ''), owned, protection, coalesce(measure_error, ''), redundancy_kind, redundancy_where, + redundancy_healthy, redundancy_said from data_item` + +func scanData(rows pgx.Row) (DataRecord, error) { + var r DataRecord + var kind, where, said *string + var healthy *bool + err := rows.Scan(&r.Machine, &r.Module, &r.Item, &r.Class, &r.Path, &r.FirstSeen, &r.DeclaredAt, &r.Size, + &r.LastWrite, &r.MeasuredAt, &r.LastBackup, &r.RetiredAt, &r.RetiredWhy, &r.DeletedAt, &r.DeletedBy, + &r.DeletedWhy, &r.Owned, &r.Protection, &r.MeasureError, &kind, &where, &healthy, &said) + if err == nil && kind != nil { + r.Redundancy = &Redundancy{Kind: *kind, Healthy: healthy} + if where != nil { + r.Redundancy.Where = *where + } + if said != nil { + r.Redundancy.Said = *said + } + } + return r, err +} + +func nullable(s string) *string { + if s == "" { + return nil + } + return &s +} + +// Data is every item the mesh keeps, by machine, module and item. +func (i *Inventory) Data(ctx context.Context) ([]DataRecord, error) { + rows, err := i.store.Pool().Query(ctx, dataSelect+` order by machine, module, item`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []DataRecord + for rows.Next() { + r, err := scanData(rows) + if err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +// DataOf is one item. +func (i *Inventory) DataOf(ctx context.Context, machine, module, item string) (DataRecord, error) { + r, err := scanData(i.store.Pool().QueryRow(ctx, dataSelect+` where machine = $1 and module = $2 and item = $3`, + machine, module, item)) + if errors.Is(err, pgx.ErrNoRows) { + return r, fmt.Errorf("the mesh knows no data %s of %s on %s", item, module, machine) + } + return r, err +} + +// DataPeaks is each item's largest reading since a moment, keyed by DataRecord.Key. +func (i *Inventory) DataPeaks(ctx context.Context, since time.Time) (map[string]int64, error) { + rows, err := i.store.Pool().Query(ctx, `select machine, module, item, max(size_bytes) from data_reading + where at >= $1 group by machine, module, item`, since) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]int64{} + for rows.Next() { + var machine, module, item string + var peak int64 + if err := rows.Scan(&machine, &module, &item, &peak); err != nil { + return nil, err + } + out[machine+"/"+module+"/"+item] = peak + } + return out, rows.Err() +} + +// MarkDataDeleted records that a person deleted a retired item. Refused for an item not retired. +func (i *Inventory) MarkDataDeleted(ctx context.Context, machine, module, item, by, why string, at time.Time) error { + tag, err := i.store.Pool().Exec(ctx, `update data_item set deleted_at = $4, deleted_by = $5, deleted_why = $6 + where machine = $1 and module = $2 and item = $3 and retired_at is not null and deleted_at is null`, + machine, module, item, at, by, why) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + return fmt.Errorf("%s of %s on %s is not retired: only retired data is deleted", item, module, machine) + } + return nil +} diff --git a/internal/inventory/data_test.go b/internal/inventory/data_test.go new file mode 100644 index 0000000..1efc142 --- /dev/null +++ b/internal/inventory/data_test.go @@ -0,0 +1,109 @@ +package inventory + +import ( + "testing" + "time" +) + +func size(n int64) *int64 { return &n } + +func at(t time.Time) *time.Time { return &t } + +// What a machine declares is kept with what its holder measured; an irreplaceable item it no longer +// declares — its module unassigned — is RETIRED and kept, never forgotten, and comes back as it was when +// declared again; anything else no longer declared is forgotten (novox/hq ADR 0233). +func TestAnUndeclaredIrreplaceableItemIsRetiredNotForgotten(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) + declared := []DeclaredData{ + {Module: "home-assistant", Item: "config", Class: "irreplaceable", Owned: true}, + {Module: "searxng", Item: "valkey", Class: "cache"}, + {Module: "ollama", Item: "models", Class: "rebuildable"}, + } + measured := map[string]map[string]Measurement{"home-assistant": {"config": {Path: "/var/lib/home-assistant/config", + Size: size(900 << 20), LastWrite: at(now.Add(-time.Minute)), MeasuredAt: at(now), LastBackup: at(now.Add(-6 * time.Hour))}}} + if _, err := inv.RecordData(ctx, "home", declared, measured, "", now); err != nil { + t.Fatal(err) + } + all, err := inv.Data(ctx) + if err != nil || len(all) != 3 { + t.Fatalf("%+v, %v", all, err) + } + + // Unassigned: nothing of it is declared any more. + later := now.Add(time.Hour) + change, err := inv.RecordData(ctx, "home", nil, nil, "home-assistant unassigned", later) + if err != nil { + t.Fatal(err) + } + if len(change.Retired) != 1 || change.Retired[0].Item != "config" { + t.Fatalf("retired %+v", change.Retired) + } + all, err = inv.Data(ctx) + if err != nil { + t.Fatal(err) + } + if len(all) != 1 || !all[0].Retired() || all[0].Path != "/var/lib/home-assistant/config" || + *all[0].Size != 900<<20 || all[0].RetiredWhy != "home-assistant unassigned" { + t.Fatalf("the irreplaceable item is not kept retired with where it is: %+v", all) + } + + // A second run that still does not declare it changes nothing: retired once, not again. + change, err = inv.RecordData(ctx, "home", nil, nil, "again", later.Add(time.Hour)) + if err != nil || len(change.Retired) != 0 { + t.Fatalf("retired twice: %+v, %v", change, err) + } + + // Deleting needs it retired; assigned again it is not retired any more, and its history stays. + if err := inv.MarkDataDeleted(ctx, "home", "searxng", "valkey", "p", "w", later); err == nil { + t.Fatal("deleting something the mesh does not hold retired was recorded") + } + change, err = inv.RecordData(ctx, "home", declared[:1], nil, "", later.Add(2*time.Hour)) + if err != nil || len(change.Reenabled) != 1 { + t.Fatalf("declared again: %+v, %v", change, err) + } + r, err := inv.DataOf(ctx, "home", "home-assistant", "config") + if err != nil || r.Retired() || !r.FirstSeen.Equal(now) || r.Size == nil { + t.Fatalf("declared again lost what was known: %+v, %v", r, err) + } + + // Retired and then deleted by a person: recorded, never by dropping the row. + if _, err := inv.RecordData(ctx, "home", nil, nil, "unassigned again", later.Add(3*time.Hour)); err != nil { + t.Fatal(err) + } + if err := inv.MarkDataDeleted(ctx, "home", "home-assistant", "config", "jochen", "moved to the anchor", later.Add(4*time.Hour)); err != nil { + t.Fatal(err) + } + r, err = inv.DataOf(ctx, "home", "home-assistant", "config") + if err != nil || r.DeletedAt == nil || r.DeletedBy != "jochen" || r.Retired() { + t.Fatalf("a deletion is not on record: %+v, %v", r, err) + } +} + +// A reading is kept at most once an hour, and the peak is read over the window asked. +func TestReadingsAreHourlyAndThePeakIsTheLargest(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + now := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC) + declared := []DeclaredData{{Module: "grafana", Item: "data", Class: "valuable", Owned: true}} + for i, s := range []int64{100, 120, 999, 130, 40} { + when := now.Add(time.Duration(i) * 20 * time.Minute) + if _, err := inv.RecordData(ctx, "ace", declared, map[string]map[string]Measurement{"grafana": {"data": { + Path: "/var/lib/grafana/data", Size: size(s), MeasuredAt: at(when)}}}, "", when); err != nil { + t.Fatal(err) + } + } + var n int + if err := inv.store.Pool().QueryRow(ctx, `select count(*) from data_reading`).Scan(&n); err != nil || n != 2 { + t.Fatalf("%d readings kept for five measurements over 80 minutes, want 2 (one an hour): %v", n, err) + } + peaks, err := inv.DataPeaks(ctx, now.Add(-time.Hour)) + if err != nil || peaks["ace/grafana/data"] != 130 { + t.Fatalf("peak %v, %v", peaks, err) + } + r, _ := inv.DataOf(ctx, "ace", "grafana", "data") + if r.Size == nil || *r.Size != 40 { + t.Fatalf("the newest measurement is not the one on record: %+v", r) + } +} diff --git a/internal/inventory/migrations/0072-a-module-declares-the-data-it-holds.sql b/internal/inventory/migrations/0072-a-module-declares-the-data-it-holds.sql new file mode 100644 index 0000000..d029f91 --- /dev/null +++ b/internal/inventory/migrations/0072-a-module-declares-the-data-it-holds.sql @@ -0,0 +1,65 @@ +-- The data a module declares, as the mesh found it on each machine (novox/hq ADR 0233). +-- +-- A module's manifest says what data it keeps and of which class. The self-check asks each machine's +-- backup holder what it measured of every declared item — its size, its last write, its last good +-- backup — and keeps that here: so a shrink is read against what the item held before, an item a +-- machine no longer declares is still known to be there, and an empty copy of an item is told from a +-- full one somewhere else. +-- +-- **Keyed by the machine's name, not a reference.** The data outlives the machine record, as a +-- binding's provider does (migration 0071): a machine leaving the mesh must not turn the record of +-- what it holds into nothing. +-- +-- **Retired, never removed.** An irreplaceable or valuable item in a module's own directory that its +-- machine no longer declares — its module unassigned — is marked retired, with when and why, and stays +-- until a person deletes it through `cleanup delete` (ADR 0230); the deletion is recorded here too, +-- never by dropping the row. An item on an operator's path (an access) is never the mesh's to retire. +-- +-- Numbered 0072, past 0071, the highest on main or any open branch when this was written. +create table data_item ( + machine text not null, + module text not null, + item text not null, + class text not null, + -- Whether it is in the module's own directory (the mesh's to retire) or an operator's path, and + -- how it is protected: backup, redundancy, both, or none. + owned boolean not null default true, + protection text not null default '', + -- Where it is on the machine, as the backup holder last said; empty until one has. + path text not null default '', + first_seen timestamptz not null default now(), + -- When a composition of the machine last declared it. + declared_at timestamptz not null default now(), + -- The newest measurement: size in bytes, the newest write inside it, and when it was measured. + size_bytes bigint, + last_write timestamptz, + measured_at timestamptz, + -- The newest good backup that covers it. + last_backup timestamptz, + -- What the holder could not measure, when it could not: the path gone, a walk refused. + measure_error text, + -- The redundant storage it is on, as the holder read it: zfs, md or btrfs, which pool or device, + -- whether it is healthy, and what it said. + redundancy_kind text, + redundancy_where text, + redundancy_healthy boolean, + redundancy_said text, + retired_at timestamptz, + retired_why text, + deleted_at timestamptz, + deleted_by text, + deleted_why text, + primary key (machine, module, item) +); + +-- One row per measurement kept, at most one an hour per item, for ninety days: what a shrink is +-- read against. +create table data_reading ( + machine text not null, + module text not null, + item text not null, + at timestamptz not null, + size_bytes bigint not null, + last_write timestamptz, + primary key (machine, module, item, at) +); diff --git a/internal/link/retirement.go b/internal/link/retirement.go index fe6b532..2f580c1 100644 --- a/internal/link/retirement.go +++ b/internal/link/retirement.go @@ -233,10 +233,13 @@ type RetirementRejected struct { // RetirementState is a provider's answer to provisioner_retirement. type RetirementState struct { - Resource string `json:"resource"` - Node string `json:"node"` - Held []string `json:"held"` - StablePasses int `json:"stable_passes"` + Resource string `json:"resource"` + Node string `json:"node"` + Held []string `json:"held"` + // HeldSizes is what each held consumer keeps on the backend, in bytes, where the backend can say + // (novox/hq ADR 0233): what an empty replacement of a consumer's data is told by. + HeldSizes map[string]int64 `json:"held_sizes,omitempty"` + StablePasses int `json:"stable_passes"` // StableForSeconds is how long the same unasked set must hold as well (ADR 0230: ten minutes). StableForSeconds int `json:"stable_for_seconds,omitempty"` // RetiresBy is "disable", or "mark-only" for a provider that cannot disable a consumer. From abf9125689bd1e11352db0444d9ffaf26698b2a2 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 6 Oct 2026 17:00:22 +0200 Subject: [PATCH 2/2] Measure each item its own way; never compare a partial size (hq ADR 0233) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A walk over a large library every hour loads the array that protects it. An item now says how it is measured — a bounded daily walk, a dataset's counters, or its top level only — and a size that is a lower bound is kept as such and never read as a shrink. --- cmd/mesh-controller/data.go | 45 ++++++++++++++++--- cmd/mesh-controller/data_test.go | 30 +++++++++++++ internal/catalogue/data.go | 27 ++++++++++- internal/catalogue/data_test.go | 9 ++-- internal/inventory/data.go | 38 ++++++++++++---- internal/inventory/data_test.go | 21 +++++++++ ...72-a-module-declares-the-data-it-holds.sql | 5 ++- 7 files changed, 156 insertions(+), 19 deletions(-) diff --git a/cmd/mesh-controller/data.go b/cmd/mesh-controller/data.go index 874e477..2399382 100644 --- a/cmd/mesh-controller/data.go +++ b/cmd/mesh-controller/data.go @@ -95,6 +95,8 @@ type holderItem struct { MeasuredAt *time.Time `json:"measured_at"` LastBackup *time.Time `json:"last_backup"` Error string `json:"error,omitempty"` + // Precision is what the size is: exact, a dataset's, partial, or none (ADR 0233). + Precision string `json:"precision,omitempty"` // Redundancy is the redundant storage the item is on, where the holder could tell (ADR 0233). Redundancy *inventory.Redundancy `json:"redundancy,omitempty"` } @@ -112,7 +114,8 @@ func readHolder(raw json.RawMessage) (map[string]map[string]inventory.Measuremen out[m.Module] = map[string]inventory.Measurement{} } out[m.Module][it.Item] = inventory.Measurement{Path: it.Path, Size: it.SizeBytes, LastWrite: it.LastWrite, - MeasuredAt: it.MeasuredAt, LastBackup: it.LastBackup, Error: it.Error, Redundancy: it.Redundancy} + MeasuredAt: it.MeasuredAt, LastBackup: it.LastBackup, Error: it.Error, Redundancy: it.Redundancy, + Precision: it.Precision} } } return out, nil @@ -377,6 +380,12 @@ func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf var out []conditions.Observation byItem := map[string][]inventory.DataRecord{} arrays := map[string][]inventory.DataRecord{} + type shrunk struct { + machine, dataset, class string + size, peak int64 + items []string + } + shrunkDatasets := map[string]shrunk{} for _, r := range records { if r.DeletedAt != nil { continue @@ -416,7 +425,18 @@ func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf Kind: kindDataMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator, Summary: fmt.Sprintf("%s of %s on %s (%s) is gone: %s does not exist any more", r.Item, r.Module, r.Machine, class, orUnknownPath(r.Path))}) - } else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && *r.Size*2 < peak && peak-*r.Size >= shrinkFloor { + } else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) && + *r.Size*2 < peak && peak-*r.Size >= shrinkFloor && inventory.Dataset(r.Precision) != "" { + // Several items on one dataset share its size: one condition for the dataset, as loud as the + // most precious item on it. + k := r.Machine + "/" + inventory.Dataset(r.Precision) + ds := shrunkDatasets[k] + ds.machine, ds.dataset, ds.size, ds.peak = r.Machine, inventory.Dataset(r.Precision), *r.Size, peak + ds.class = catalogue.StricterClass(ds.class, class) + ds.items = append(ds.items, r.Module+"/"+r.Item) + shrunkDatasets[k] = ds + } else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) && + *r.Size*2 < peak && peak-*r.Size >= shrinkFloor { out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataShrank, Kind: kindDataShrank, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator, Summary: fmt.Sprintf("%s of %s on %s (%s) shrank to %s from %s within %d days — more than half of what it "+ @@ -453,6 +473,15 @@ func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf } } } + for _, k := range keysSorted(shrunkDatasets) { + ds := shrunkDatasets[k] + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, + ID: ds.machine + ".dataset." + strings.ReplaceAll(ds.dataset, "/", "-"), Token: kindDataShrank, + Kind: kindDataShrank, Machine: ds.machine, Severity: severityOf(ds.class), Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("the dataset %s on %s shrank to %s from %s within %d days — more than half of what it held "+ + "is gone; it holds %s", ds.dataset, ds.machine, sizeWords(&ds.size), sizeWords(&ds.peak), + int(shrinkWindow.Hours()/24), strings.Join(ds.items, ", "))}) + } // The redundant storage watched data is on: one condition per array, as loud as the most precious // item on it — the array, not each item, is what degrades. for _, where := range keysSorted(arrays) { @@ -484,11 +513,12 @@ func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf for _, key := range keysSorted(byItem) { rs := byItem[key] for _, a := range rs { - if a.Retired() || a.Size == nil || !catalogue.Watched(a.Class) { + if a.Retired() || a.Size == nil || !catalogue.Watched(a.Class) || !inventory.Comparable(a.Precision) { continue } for _, o := range rs { - if o.Machine == a.Machine || !o.Retired() || o.Size == nil || !replacedByLess(*a.Size, *o.Size) { + if o.Machine == a.Machine || !o.Retired() || o.Size == nil || !inventory.Comparable(o.Precision) || + !replacedByLess(*a.Size, *o.Size) { continue } out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, @@ -603,6 +633,8 @@ type dataRow struct { // Array is the redundant storage it is on and its state, where its holder could tell. Array string `json:"array,omitempty"` Unmeasured string `json:"unmeasured,omitempty"` + // Precision says what the size is: exact, a dataset's whole size, partial, or none. + Precision string `json:"precision,omitempty"` SizeBytes *int64 `json:"size-bytes,omitempty"` LastWrite string `json:"last-write,omitempty"` MeasuredAt string `json:"measured-at,omitempty"` @@ -659,6 +691,9 @@ func dataCommand(ctx context.Context, args []string) error { if r.Array != "" { fmt.Printf(" on %s\n", r.Array) } + if r.Precision != "" && r.Precision != "exact" { + fmt.Printf(" size: %s\n", r.Precision) + } if r.Unmeasured != "" { fmt.Printf(" not measured: %s\n", r.Unmeasured) } @@ -687,7 +722,7 @@ func dataRows(records []inventory.DataRecord, shelf map[string]catalogue.Manifes continue } row := dataRow{Machine: r.Machine, Module: r.Module, Item: r.Item, Class: r.Class, Path: r.Path, - Protection: r.Protection, Unmeasured: r.MeasureError, SizeBytes: r.Size, LastWrite: stamp(r.LastWrite), MeasuredAt: stamp(r.MeasuredAt), + Protection: r.Protection, Unmeasured: r.MeasureError, Precision: r.Precision, SizeBytes: r.Size, LastWrite: stamp(r.LastWrite), MeasuredAt: stamp(r.MeasuredAt), LastBackup: stamp(r.LastBackup), Retired: stamp(r.RetiredAt), RetiredWhy: r.RetiredWhy, Deleted: stamp(r.DeletedAt)} if it, ok := shelf[r.Module].DataItem(r.Item); ok && it.BackedUp() { diff --git a/cmd/mesh-controller/data_test.go b/cmd/mesh-controller/data_test.go index 9fc3f00..2ed87e2 100644 --- a/cmd/mesh-controller/data_test.go +++ b/cmd/mesh-controller/data_test.go @@ -428,3 +428,33 @@ func TestKeptByHoldsTheProvidersItemToTheConsumersClass(t *testing.T) { t.Fatalf("a valuable store without a backup: %+v", o) } } + +// Items measured from one dataset's counters share its size: a shrink of the dataset is one condition +// naming every item on it, not one per item; and a partial walk's lower bound is never compared. +func TestADatasetShrinksOnceAndAPartialSizeIsNeverCompared(t *testing.T) { + now := time.Now() + media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}], + "data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array","measure":"dataset"}, + {"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array","measure":"dataset"}]}}` + shelf := shelfFor(t, media, houseManifest) + well := true + on := func(item string, size int64) inventory.DataRecord { + return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable", + Size: bytesOf(size), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now), + Precision: "dataset tank/media: its whole size", + Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: &well}} + } + films, shows := on("films", 30<<40), on("shows", 30<<40) + peaks := map[string]int64{films.Key(): 90 << 40, shows.Key(): 90 << 40} + got := dataFindings([]inventory.DataRecord{films, shows}, peaks, shelf, nil, nil, now) + if len(got) != 1 || got[0].Kind != kindDataShrank || got[0].Severity != conditions.Urgent || + !strings.Contains(got[0].Summary, "media/films, media/shows") { + t.Fatalf("%+v", got) + } + partial := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", + Size: bytesOf(1 << 20), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now), LastWrite: when(now), + LastBackup: when(now), Precision: "partial: measured partially"} + if got := dataFindings([]inventory.DataRecord{partial}, map[string]int64{partial.Key(): 1 << 30}, shelf, nil, nil, now); len(got) != 0 { + t.Fatalf("a partial size was compared: %+v", got) + } +} diff --git a/internal/catalogue/data.go b/internal/catalogue/data.go index 26f2a8e..e44ac75 100644 --- a/internal/catalogue/data.go +++ b/internal/catalogue/data.go @@ -102,6 +102,10 @@ type DataItem struct { Redundancy string `json:"redundancy,omitempty"` // Within is how old its last good backup may be; unsaid, DefaultBackupWithin. Within string `json:"within,omitempty"` + // Measure is how the backup holder measures it: `walk` (the default — every file, at most daily and + // bounded, for small items), `dataset` (a ZFS dataset's own counters, hourly, nothing walked) or + // `shallow` (its top-level entries only, no size). A large item never says walk. + Measure string `json:"measure,omitempty"` // Active is how long it may go unwritten before that is a fault — for data something is // expected to write all the time. Unsaid, a quiet item is not a fault. Active string `json:"active,omitempty"` @@ -207,6 +211,21 @@ func (it DataItem) Protection() string { return strings.Join(by, "+") } +// The ways an item is measured. +const ( + MeasureWalk = "walk" + MeasureDataset = "dataset" + MeasureShallow = "shallow" +) + +// MeasuredBy is how the item is measured, with the default applied. +func (it DataItem) MeasuredBy() string { + if it.Measure == "" { + return MeasureWalk + } + return it.Measure +} + // OwnedByModule is whether the item is in one of the module's own directories — the mesh's to retire — // rather than an operator's path it was given, which the mesh never retires and never deletes. func (it DataItem) OwnedByModule() bool { return strings.HasPrefix(it.Path, "${dir:") } @@ -365,6 +384,11 @@ func (m Manifest) dataProblems() []string { if it.Class == ClassCache && it.Redundancy != "" { say("%s is a cache and says it is protected by redundancy; a cache is not protected", label) } + switch it.Measure { + case "", MeasureWalk, MeasureDataset, MeasureShallow: + default: + say("%s's measure %q is walk, dataset or shallow", label, it.Measure) + } if _, err := ParseDataDuration(it.Within); err != nil { say("%s's within: %v", label, err) } @@ -541,7 +565,8 @@ func (m Manifest) derivedContributions() []SeatContribution { case it.BackedUp(): covered = it.Path } - described = append(described, fmt.Sprintf("item %s %s %s %s %s", it.ID, it.Class, it.Path, covered, it.Protection())) + described = append(described, fmt.Sprintf("item %s %s %s %s %s %s", it.ID, it.Class, it.Path, covered, + it.Protection(), it.MeasuredBy())) } var out []SeatContribution if len(lines) > 0 { diff --git a/internal/catalogue/data_test.go b/internal/catalogue/data_test.go index e083f49..6e55f21 100644 --- a/internal/catalogue/data_test.go +++ b/internal/catalogue/data_test.go @@ -55,6 +55,7 @@ func TestTheDataSectionIsRefusedWhereItIsWrong(t *testing.T) { {"irreplaceable and protected by nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":"none"}]}`, "protected one way or the other"}, {"a cache on redundancy", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","redundancy":"an array"}]}`, "a cache is not protected"}, {"kept with something not required", `{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}`, "does not require"}, + {"a measure it does not know", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","measure":"du"}]}`, "walk, dataset or shallow"}, {"a cache expecting writes", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","active":"1d"}]}`, "only irreplaceable and valuable"}, {"a machine path", `{"own":[{"id":"a","path":"/srv/a","class":"cache"}]}`, "names no machine path"}, {"a directory it does not declare", `{"own":[{"id":"a","path":"${dir:nowhere}","class":"cache"}]}`, "declares no directory"}, @@ -168,7 +169,7 @@ func TestTheBackupHoldersLinesAreDerivedFromTheData(t *testing.T) { "resources":[{"id":"home","type":"directory","path":"${machine:account-home}/.agent","mode":"0700"}]}`) media := mustParse(t, `{"module":"media","version":"1", "accesses":[{"id":"films","mode":"read"}], - "data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"on a redundant array; no room for a copy"}, + "data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"on a redundant array; no room for a copy","measure":"dataset"}, {"id":"meta","path":"${dir:meta}","class":"rebuildable"}]}, "resources":[{"id":"meta","type":"directory","mode":"0700"}]}`) facts := map[string]string{"account-home": "/home/op"} @@ -185,9 +186,9 @@ func TestTheBackupHoldersLinesAreDerivedFromTheData(t *testing.T) { if err != nil { t.Fatal(err) } - want = "# agent\nitem home valuable /home/op/.agent /home/op/.agent backup\nitem tmp cache /home/op/.agent/tmp - none\n" + - "# media\nitem films irreplaceable /tank/films - redundancy\nitem meta rebuildable /var/lib/media/meta /var/lib/media/meta backup\n" + - "# pg\nitem store irreplaceable /srv/store /srv/store/dumps backup\nitem dumps rebuildable /srv/store/dumps - none\n" + want = "# agent\nitem home valuable /home/op/.agent /home/op/.agent backup walk\nitem tmp cache /home/op/.agent/tmp - none walk\n" + + "# media\nitem films irreplaceable /tank/films - redundancy dataset\nitem meta rebuildable /var/lib/media/meta /var/lib/media/meta backup walk\n" + + "# pg\nitem store irreplaceable /srv/store /srv/store/dumps backup walk\nitem dumps rebuildable /srv/store/dumps - none walk\n" if data != want { t.Fatalf("data lines:\n%s\nwant:\n%s", data, want) } diff --git a/internal/inventory/data.go b/internal/inventory/data.go index f7a7645..3518e57 100644 --- a/internal/inventory/data.go +++ b/internal/inventory/data.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "time" "github.com/jackc/pgx/v5" @@ -43,7 +44,9 @@ type Measurement struct { MeasuredAt *time.Time LastBackup *time.Time // Error is what the holder could not measure, when it could not. - Error string + Error string + // Precision is what the size is, as the holder said it: "exact", "dataset …", "partial: …", "no size: …". + Precision string Redundancy *Redundancy } @@ -54,6 +57,7 @@ type DataRecord struct { Protection string FirstSeen, DeclaredAt time.Time MeasureError string + Precision string Redundancy *Redundancy Size *int64 LastWrite, MeasuredAt, LastBackup *time.Time @@ -63,6 +67,21 @@ type DataRecord struct { DeletedBy, DeletedWhy string } +// Comparable is whether a size is fit to compare with another: exact, or a dataset's own counters. +func Comparable(precision string) bool { + return precision == "" || precision == "exact" || strings.HasPrefix(precision, "dataset ") +} + +// Dataset is the ZFS dataset a size was read from, when it was: what several items on one dataset share. +func Dataset(precision string) string { + rest, ok := strings.CutPrefix(precision, "dataset ") + if !ok { + return "" + } + name, _, _ := strings.Cut(rest, ":") + return name +} + // Retired is whether the item is retired and not deleted. func (r DataRecord) Retired() bool { return r.RetiredAt != nil && r.DeletedAt == nil } @@ -139,12 +158,15 @@ func (i *Inventory) RecordData(ctx context.Context, machine string, declared []D if _, err := tx.Exec(ctx, ` insert into data_item (machine, module, item, class, path, first_seen, declared_at, size_bytes, last_write, measured_at, last_backup, owned, protection, - measure_error, redundancy_kind, redundancy_where, redundancy_healthy, redundancy_said) - values ($1, $2, $3, $4, $5, $6, $6, $7, $8, $9, $10, $12, $13, $14, $15, $16, $17, $18) + measure_error, redundancy_kind, redundancy_where, redundancy_healthy, redundancy_said, + precision) + values ($1, $2, $3, $4, $5, $6, $6, $7, $8, $9, $10, $12, $13, $14, $15, $16, $17, $18, $19) on conflict (machine, module, item) do update set class = excluded.class, owned = excluded.owned, protection = excluded.protection, + precision = case when excluded.measured_at is not null then excluded.precision else data_item.precision end, + size_bytes = case when excluded.measured_at is not null then excluded.size_bytes else data_item.size_bytes end, measure_error = case when excluded.measured_at is not null then excluded.measure_error else data_item.measure_error end, redundancy_kind = case when excluded.measured_at is not null then excluded.redundancy_kind else data_item.redundancy_kind end, redundancy_where = case when excluded.measured_at is not null then excluded.redundancy_where else data_item.redundancy_where end, @@ -153,17 +175,17 @@ func (i *Inventory) RecordData(ctx context.Context, machine string, declared []D path = case when excluded.path <> '' then excluded.path else data_item.path end, first_seen = case when $11::boolean then excluded.first_seen else data_item.first_seen end, declared_at = excluded.declared_at, - size_bytes = coalesce(excluded.size_bytes, data_item.size_bytes), last_write = coalesce(excluded.last_write, data_item.last_write), measured_at = coalesce(excluded.measured_at, data_item.measured_at), last_backup = coalesce(excluded.last_backup, data_item.last_backup), retired_at = null, retired_why = null, deleted_at = null, deleted_by = null, deleted_why = null`, machine, d.Module, d.Item, d.Class, m.Path, now, m.Size, m.LastWrite, m.MeasuredAt, m.LastBackup, - fresh, d.Owned, d.Protection, nullable(m.Error), red.Kind, red.Where, red.Healthy, red.Said); err != nil { + fresh, d.Owned, d.Protection, nullable(m.Error), red.Kind, red.Where, red.Healthy, red.Said, + nullable(m.Precision)); err != nil { return change, err } - if m.Size != nil && m.MeasuredAt != nil { + if m.Size != nil && m.MeasuredAt != nil && Comparable(m.Precision) { if _, err := tx.Exec(ctx, ` insert into data_reading (machine, module, item, at, size_bytes, last_write) select $1, $2, $3, $4, $5, $6 @@ -203,7 +225,7 @@ func (i *Inventory) RecordData(ctx context.Context, machine string, declared []D const dataSelect = `select machine, module, item, class, path, first_seen, declared_at, size_bytes, last_write, measured_at, last_backup, retired_at, coalesce(retired_why, ''), deleted_at, coalesce(deleted_by, ''), coalesce(deleted_why, ''), owned, protection, coalesce(measure_error, ''), redundancy_kind, redundancy_where, - redundancy_healthy, redundancy_said from data_item` + redundancy_healthy, redundancy_said, coalesce(precision, '') from data_item` func scanData(rows pgx.Row) (DataRecord, error) { var r DataRecord @@ -211,7 +233,7 @@ func scanData(rows pgx.Row) (DataRecord, error) { var healthy *bool err := rows.Scan(&r.Machine, &r.Module, &r.Item, &r.Class, &r.Path, &r.FirstSeen, &r.DeclaredAt, &r.Size, &r.LastWrite, &r.MeasuredAt, &r.LastBackup, &r.RetiredAt, &r.RetiredWhy, &r.DeletedAt, &r.DeletedBy, - &r.DeletedWhy, &r.Owned, &r.Protection, &r.MeasureError, &kind, &where, &healthy, &said) + &r.DeletedWhy, &r.Owned, &r.Protection, &r.MeasureError, &kind, &where, &healthy, &said, &r.Precision) if err == nil && kind != nil { r.Redundancy = &Redundancy{Kind: *kind, Healthy: healthy} if where != nil { diff --git a/internal/inventory/data_test.go b/internal/inventory/data_test.go index 1efc142..d109c1b 100644 --- a/internal/inventory/data_test.go +++ b/internal/inventory/data_test.go @@ -107,3 +107,24 @@ func TestReadingsAreHourlyAndThePeakIsTheLargest(t *testing.T) { t.Fatalf("the newest measurement is not the one on record: %+v", r) } } + +// A partial walk's lower bound is kept as the newest measurement, said as partial, and never kept as a +// reading a shrink would be read against. +func TestAPartialMeasurementIsNeverAReading(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + now := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC) + declared := []DeclaredData{{Module: "big", Item: "data", Class: "valuable", Owned: true}} + if _, err := inv.RecordData(ctx, "home", declared, map[string]map[string]Measurement{"big": {"data": { + Path: "/srv/big", Size: size(5), MeasuredAt: at(now), Precision: "partial: stopped"}}}, "", now); err != nil { + t.Fatal(err) + } + var n int + if err := inv.store.Pool().QueryRow(ctx, `select count(*) from data_reading`).Scan(&n); err != nil || n != 0 { + t.Fatalf("%d readings from a partial measurement: %v", n, err) + } + r, err := inv.DataOf(ctx, "home", "big", "data") + if err != nil || r.Precision != "partial: stopped" || Comparable(r.Precision) { + t.Fatalf("%+v, %v", r, err) + } +} diff --git a/internal/inventory/migrations/0072-a-module-declares-the-data-it-holds.sql b/internal/inventory/migrations/0072-a-module-declares-the-data-it-holds.sql index d029f91..0e29e93 100644 --- a/internal/inventory/migrations/0072-a-module-declares-the-data-it-holds.sql +++ b/internal/inventory/migrations/0072-a-module-declares-the-data-it-holds.sql @@ -38,6 +38,8 @@ create table data_item ( last_backup timestamptz, -- What the holder could not measure, when it could not: the path gone, a walk refused. measure_error text, + -- What the newest size is: exact, a dataset's whole size, partial (a lower bound) or none. + precision text, -- The redundant storage it is on, as the holder read it: zfs, md or btrfs, which pool or device, -- whether it is healthy, and what it said. redundancy_kind text, @@ -53,7 +55,8 @@ create table data_item ( ); -- One row per measurement kept, at most one an hour per item, for ninety days: what a shrink is --- read against. +-- read against. Only a comparable size is kept: exact, or a dataset's own counters — never a partial +-- walk's lower bound. create table data_reading ( machine text not null, module text not null,