diff --git a/cmd/mesh-controller/acts.go b/cmd/mesh-controller/acts.go index bcd4626..5ad5f70 100644 --- a/cmd/mesh-controller/acts.go +++ b/cmd/mesh-controller/acts.go @@ -224,6 +224,10 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string) for _, line := range unheldChange(shelf, node, assigned, left) { answer += "\n " + line } + // What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233). + for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) { + answer += "\n " + line + } return answer + blockedElsewhere(ctx, open, node), nil } diff --git a/cmd/mesh-controller/check.go b/cmd/mesh-controller/check.go index fe1ed88..463af4d 100644 --- a/cmd/mesh-controller/check.go +++ b/cmd/mesh-controller/check.go @@ -93,6 +93,15 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { } failed += len(identities) + // And the data each module keeps (novox/hq ADR 0233): a provider that grants says what it keeps for + // its consumers, a directory a container writes is declared, and no backup line is written by hand. + data := catalogue.DataProblems(shelf) + sort.Strings(data) + for _, p := range data { + fmt.Fprintln(out, p) + } + failed += len(data) + var names []string for name := range shelf { names = append(names, name) @@ -121,6 +130,14 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { if len(m.Reads) > 0 { fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", ")) } + // The data it keeps, by class, so a reviewer sees what the mesh will protect and how. + if items := m.DataItems(); len(items) > 0 { + kept := make([]string, 0, len(items)) + for _, it := range items { + kept = append(kept, it.ID+" ("+it.Class+")") + } + fmt.Fprintf(out, ", keeps %s", strings.Join(kept, ", ")) + } fmt.Fprintln(out) } if failed > 0 { diff --git a/cmd/mesh-controller/data.go b/cmd/mesh-controller/data.go new file mode 100644 index 0000000..2399382 --- /dev/null +++ b/cmd/mesh-controller/data.go @@ -0,0 +1,918 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "log" + "sort" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A module declares the data it holds, and the mesh protects and watches it from that declaration +// (novox/hq ADR 0233). +// +// The self-check's D13 composes what every machine declares, asks each machine's backup holder what +// it measured of every item — size, newest write, newest good backup, the redundant storage it is on — +// and keeps both. From that, and from what every provider says it holds for its consumers, it raises, +// each URGENT for what is irreplaceable and a WARNING for what is valuable (the operator's ranking): +// +// - `data-shrank`: an item holds less than half of its largest size in seven days, and at least +// shrinkFloor less; `data-missing`: its path is gone; +// - `empty-replacement`: an item, or a consumer's data at a provider, is less than half the size of a +// copy of the same thing kept elsewhere — on 2026-10-05 five applications ran for twenty hours on +// empty databases while their real ones sat on another machine (issue 273); +// - `data-held-twice` (warning): a consumer has active data at two providers and their sizes cannot +// be compared; +// - `data-quiet`: an item said to be written all the time has not been, within its bound; +// - `backup-stale`: an item's newest good backup is older than its bound, or there is none; +// - `array-degraded`: the redundant storage an item is on is not healthy, or cannot be read; +// `protection-missing`: an item said to be protected by redundancy is on storage that is not; +// - `cleanup-waiting` (warning): an item retired more than thirty days, waiting for a person. +// +// And it retires: an irreplaceable or valuable item in a module's own directory that its machine no +// longer declares — its module unassigned — is kept, marked retired with when and why, and listed by +// `cleanup list` until `cleanup delete` removes it. The node-engine never deletes a directory with +// anything in it; this is the record of what it kept. An operator's path is never retired or deleted. + +// The condition kinds of D13. +const ( + kindDataShrank = "data-shrank" + kindEmptyReplacement = "empty-replacement" + kindDataHeldTwice = "data-held-twice" + kindDataQuiet = "data-quiet" + kindBackupStale = "backup-stale" + kindDataUnmeasured = "data-unmeasured" + // kindDataMissing is a watched item whose path is gone. + kindDataMissing = "data-missing" + // kindArrayDegraded is redundant storage watched data is on that is not healthy, or cannot be read. + kindArrayDegraded = "array-degraded" + // kindProtectionMissing is an item said to be protected by redundancy, on storage that is not. + kindProtectionMissing = "protection-missing" +) + +// probeDataID is the self-check's id for this probe. +const probeDataID = "D13" + +// The bounds the findings are read against. +var ( + // shrinkWindow is how far back the largest size is looked for. + shrinkWindow = 7 * 24 * time.Hour + // shrinkFloor is the least loss that is worth saying: two empty databases differ by a few + // megabytes, and half of almost nothing is noise. + shrinkFloor int64 = 16 << 20 + // dataAsk is how long one machine's holder, or one provider, is given to answer. + dataAsk = 8 * time.Second +) + +// keyOfItem is one item's condition id: its machine, module and item. +func keyOfItem(machine, module, item string) string { return machine + "." + module + "." + item } + +// holderAnswer is what a node-backup holder's `backed-up` says of one module (ADR 0233 adds Data). +type holderAnswer struct { + Module string `json:"module"` + Data []holderItem `json:"data"` +} + +// holderItem is one item as the holder measured it. +type holderItem struct { + Item string `json:"item"` + Class string `json:"class"` + Path string `json:"path"` + SizeBytes *int64 `json:"size_bytes"` + LastWrite *time.Time `json:"last_write"` + MeasuredAt *time.Time `json:"measured_at"` + LastBackup *time.Time `json:"last_backup"` + Error string `json:"error,omitempty"` + // Precision is what the size is: exact, a dataset's, partial, or none (ADR 0233). + Precision string `json:"precision,omitempty"` + // Redundancy is the redundant storage the item is on, where the holder could tell (ADR 0233). + Redundancy *inventory.Redundancy `json:"redundancy,omitempty"` +} + +// readHolder reads a holder's answer into measurements by module and item. +func readHolder(raw json.RawMessage) (map[string]map[string]inventory.Measurement, error) { + var modules []holderAnswer + if err := json.Unmarshal(raw, &modules); err != nil { + return nil, fmt.Errorf("its answer is not readable: %w", err) + } + out := map[string]map[string]inventory.Measurement{} + for _, m := range modules { + for _, it := range m.Data { + if out[m.Module] == nil { + out[m.Module] = map[string]inventory.Measurement{} + } + out[m.Module][it.Item] = inventory.Measurement{Path: it.Path, Size: it.SizeBytes, LastWrite: it.LastWrite, + MeasuredAt: it.MeasuredAt, LastBackup: it.LastBackup, Error: it.Error, Redundancy: it.Redundancy, + Precision: it.Precision} + } + } + return out, nil +} + +// declaredOn is every data item a machine's composition declares, and whether something there holds +// node-backup to measure them. +func declaredOn(plan catalogue.Resolution) ([]inventory.DeclaredData, bool) { + var out []inventory.DeclaredData + held := false + for _, m := range plan.Modules { + for _, c := range m.Claims { + if s, known := catalogue.SeatNamed(c.Name); known && s.Name == catalogue.BackupSeat { + held = true + } + } + for _, it := range m.DataItems() { + out = append(out, inventory.DeclaredData{Module: m.Module, Item: it.ID, Class: it.Class, + Owned: it.OwnedByModule(), Protection: it.Protection()}) + } + } + return out, held +} + +// consumerCopy is one provider's account of one consumer: where, how big, and whether still active. +type consumerCopy struct { + Node, Module, Consumer string + Size *int64 + Retired bool + // Class is how precious the consumer's data is: the stricter of what the provider keeps for its + // consumers and what the consumer says it keeps there (`kept-by`). + Class string +} + +// probeData is D13. +func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error) { + if d.js == nil { + return nil, errors.New("no bus to ask the machines over") + } + open := d.open + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + return nil, err + } + nodes, err := open.inventory.Nodes(ctx) + if err != nil { + return nil, err + } + heard := heardMachines(d) + now := time.Now() + + type machine struct { + name string + declared []inventory.DeclaredData + held bool + measured map[string]map[string]inventory.Measurement + askErr error + } + var machines []*machine + for _, n := range nodes { + plan, _, err := planFor(ctx, open, n.Name) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + // A machine that cannot be worked out declares nothing this run — which is not the same as + // declaring nothing: retiring its data on that would be acting on an unreadable result. + continue + } + declared, held := declaredOn(plan) + machines = append(machines, &machine{name: n.Name, declared: declared, held: held}) + } + // Every holder asked at once, as D8 asks every ban list. + var wg sync.WaitGroup + for _, m := range machines { + if !m.held || !heard[m.name] { + continue + } + wg.Add(1) + go func(m *machine) { + defer wg.Done() + asking, cancel := context.WithTimeout(ctx, dataAsk) + defer cancel() + raw, err := askSeatTool(asking, d.js.Conn(), catalogue.BackupSeat, "backed-up", m.name) + if err == nil { + m.measured, err = readHolder(raw) + } + m.askErr = err + }(m) + } + wg.Wait() + + var out []conditions.Observation + for _, m := range machines { + if m.askErr != nil { + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Token: kindDataUnmeasured, + Kind: kindDataUnmeasured, Machine: m.name, Severity: conditions.Warning, + Summary: fmt.Sprintf("%s's backup holder did not say what it measured of the data declared there, so "+ + "nothing about that data is known this run: %s", m.name, firstLine(m.askErr.Error())), + Said: firstLine(m.askErr.Error())}) + } + why := fmt.Sprintf("no longer declared on %s: its module was unassigned there, or is no longer pulled in", m.name) + change, err := open.inventory.RecordData(ctx, m.name, m.declared, m.measured, why, now) + if err != nil { + return nil, fmt.Errorf("what %s holds could not be kept: %w", m.name, err) + } + for _, r := range change.Retired { + log.Printf("data: %s of %s on %s RETIRED, kept at %s: %s — `cleanup list` shows it, and only `cleanup "+ + "delete` removes it (novox/hq ADR 0233)", r.Item, r.Module, r.Machine, orUnknownPath(r.Path), why) + } + for _, r := range change.Reenabled { + log.Printf("data: %s of %s on %s is declared again, no longer retired", r.Item, r.Module, r.Machine) + } + } + + records, err := open.inventory.Data(ctx) + if err != nil { + return nil, err + } + peaks, err := open.inventory.DataPeaks(ctx, now.Add(-shrinkWindow)) + if err != nil { + return nil, err + } + bindings, err := open.inventory.Bindings(ctx) + if err != nil { + return nil, err + } + upgraded, keptBy := keptByClasses(bindings, shelf) + copies, err := consumerCopies(ctx, d.js.Conn(), open.inventory, shelf, keptBy) + if err != nil { + return nil, err + } + out = append(out, dataFindings(records, peaks, shelf, copies, upgraded, now)...) + return out, nil +} + +func orUnknownPath(p string) string { + if p == "" { + return "a path its backup holder never named" + } + return p +} + +// consumerCopies asks every provider of a provision whose consumers' data is kept what it holds, at +// once. One that cannot answer is passed over: it says nothing about any copy, which is not a finding. +func consumerCopies(ctx context.Context, conn *nats.Conn, inv *inventory.Inventory, + shelf map[string]catalogue.Manifest, keptBy map[string]string) ([]consumerCopy, error) { + instances, err := providerInstances(ctx, inv) + if err != nil { + return nil, err + } + var asked []providerInstance + for _, p := range instances { + m := shelf[p.Module] + keeps := false + for provision := range m.Grants { + keeps = keeps || m.KeepsConsumerData(provision) + } + if keeps { + asked = append(asked, p) + } + } + states := make([]*link.RetirementState, len(asked)) + var wg sync.WaitGroup + for i, p := range asked { + wg.Add(1) + go func(i int, p providerInstance) { + defer wg.Done() + asking, cancel := context.WithTimeout(ctx, dataAsk) + defer cancel() + if s, err := askRetirement(asking, conn, p); err == nil { + states[i] = &s + } + }(i, p) + } + wg.Wait() + var out []consumerCopy + for i, p := range asked { + s := states[i] + if s == nil { + continue + } + class := consumersClass(shelf[p.Module]) + for _, c := range s.Held { + cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: c, + Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+c])} + if size, ok := s.HeldSizes[c]; ok && size >= 0 { + size := size + cp.Size = &size + } + out = append(out, cp) + } + for _, r := range s.Retired { + if r.Kind != "" && r.Kind != "consumer" { + continue + } + cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: r.Consumer, Retired: true, + Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+r.Consumer])} + if r.SizeBytes != nil && *r.SizeBytes >= 0 { + cp.Size = r.SizeBytes + } + out = append(out, cp) + } + } + return out, nil +} + +// severityOf is how loud a finding about data of a class is: urgent for what is irreplaceable, a warning +// for anything else watched (the operator's ranking, ADR 0233). +func severityOf(class string) conditions.Severity { + if class == catalogue.ClassIrreplaceable { + return conditions.Urgent + } + return conditions.Warning +} + +// consumersClass is the most precious class a provider keeps any of its consumers' data as. +func consumersClass(m catalogue.Manifest) string { + class := catalogue.ClassNone + for provision := range m.Grants { + if c, ok := m.ConsumerDataOf(provision); ok { + class = catalogue.StricterClass(class, c.Class) + } else if m.KeepsConsumerData(provision) { + class = catalogue.StricterClass(class, catalogue.ClassValuable) + } + } + return class +} + +// keptByClasses is what consumers say of the data they keep with their providers (`kept-by`), read +// through where each is bound: by provider module and consumer identity, the class of that consumer's +// data there; and by provider item key (machine/module/item), the class the item holding it is held to. +func keptByClasses(bindings []inventory.Binding, shelf map[string]catalogue.Manifest) (map[string]string, map[string]string) { + upgraded, keptBy := map[string]string{}, map[string]string{} + for _, b := range bindings { + m, ok := shelf[b.Consumer] + if !ok { + continue + } + k, said := m.KeptByOf(b.Provision) + if !said { + continue + } + identity := catalogue.ConsumerIdentity(b.Machine, catalogue.IdentitySource(m.Slug, m.Module)) + key := b.Provider.Module + "/" + identity + keptBy[key] = catalogue.StricterClass(keptBy[key], k.Class) + if pc, ok := shelf[b.Provider.Module].ConsumerDataOf(b.Provision); ok && pc.In != "" { + if _, own := shelf[b.Provider.Module].DataItem(pc.In); own { + item := b.Provider.Node + "/" + b.Provider.Module + "/" + pc.In + upgraded[item] = catalogue.StricterClass(upgraded[item], k.Class) + } + } + } + return upgraded, keptBy +} + +// dataFindings is every condition the data on record raises now. A function of what is known, so the +// incident's shape is tested without a mesh. upgraded is the class an item is held to where a consumer +// of its module keeps data in it more precious than its own class says (`kept-by`), by its key. +func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf map[string]catalogue.Manifest, + copies []consumerCopy, upgraded map[string]string, now time.Time) []conditions.Observation { + var out []conditions.Observation + byItem := map[string][]inventory.DataRecord{} + arrays := map[string][]inventory.DataRecord{} + type shrunk struct { + machine, dataset, class string + size, peak int64 + items []string + } + shrunkDatasets := map[string]shrunk{} + for _, r := range records { + if r.DeletedAt != nil { + continue + } + class := catalogue.StricterClass(r.Class, upgraded[r.Key()]) + r.Class = class + byItem[r.Module+"/"+r.Item] = append(byItem[r.Module+"/"+r.Item], r) + item, declared := shelf[r.Module].DataItem(r.Item) + id := keyOfItem(r.Machine, r.Module, r.Item) + if r.Retired() { + if now.Sub(*r.RetiredAt) > cleanupAfter { + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "cleanup", + Kind: kindCleanupWaiting, Machine: r.Machine, Severity: conditions.Warning, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept at %s since %s; `cleanup "+ + "delete %s %s %s --why …` once a person has decided, or assign %s there again", + r.Item, r.Module, r.Machine, r.Class, sizeWords(r.Size), int(now.Sub(*r.RetiredAt).Hours()/24), + orUnknownPath(r.Path), r.RetiredWhy, r.Machine, r.Module, r.Item, r.Module)}) + } + continue + } + if !catalogue.Watched(class) { + continue + } + severity := severityOf(class) + if r.Redundancy != nil { + where := r.Machine + "/" + r.Redundancy.Kind + ":" + r.Redundancy.Where + arrays[where] = append(arrays[where], r) + } else if declared && item.Redundancy != "" && r.MeasuredAt != nil && r.MeasureError == "" { + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindProtectionMissing, + Kind: kindProtectionMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s of %s on %s (%s) is said to be protected by the redundancy of the storage it is on, "+ + "and %s is on nothing the backup holder can read as redundant: it has no protection the mesh can see", + r.Item, r.Module, r.Machine, class, orUnknownPath(r.Path))}) + } + if r.MeasureError != "" && strings.Contains(r.MeasureError, "does not exist") { + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataMissing, + Kind: kindDataMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s of %s on %s (%s) is gone: %s does not exist any more", r.Item, r.Module, r.Machine, + class, orUnknownPath(r.Path))}) + } else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) && + *r.Size*2 < peak && peak-*r.Size >= shrinkFloor && inventory.Dataset(r.Precision) != "" { + // Several items on one dataset share its size: one condition for the dataset, as loud as the + // most precious item on it. + k := r.Machine + "/" + inventory.Dataset(r.Precision) + ds := shrunkDatasets[k] + ds.machine, ds.dataset, ds.size, ds.peak = r.Machine, inventory.Dataset(r.Precision), *r.Size, peak + ds.class = catalogue.StricterClass(ds.class, class) + ds.items = append(ds.items, r.Module+"/"+r.Item) + shrunkDatasets[k] = ds + } else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) && + *r.Size*2 < peak && peak-*r.Size >= shrinkFloor { + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataShrank, + Kind: kindDataShrank, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s of %s on %s (%s) shrank to %s from %s within %d days — more than half of what it "+ + "held is gone. If that was meant, silence this with why; if not, `node-backup.restore` puts the last "+ + "good copy beside it", r.Item, r.Module, r.Machine, class, sizeWords(r.Size), sizeWords(&peak), + int(shrinkWindow.Hours()/24))}) + } + if !declared { + continue + } + if within := item.ActiveWithin(); within > 0 && r.LastWrite != nil && now.Sub(*r.LastWrite) > within { + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataQuiet, + Kind: kindDataQuiet, Machine: r.Machine, Severity: severity, + Summary: fmt.Sprintf("%s of %s on %s is written all the time, and has not been since %s (its bound is %s): "+ + "whatever writes it has stopped", r.Item, r.Module, r.Machine, r.LastWrite.UTC().Format(time.RFC3339), + within)}) + } + // A backup is required of what is irreplaceable and copied; of what is valuable it is the standard + // plan, said only where the machine was measured — where a holder is there to take it. + if item.BackedUp() && (class == catalogue.ClassIrreplaceable || r.MeasuredAt != nil) { + within := item.BackupWithin() + switch { + case r.LastBackup == nil && now.Sub(r.FirstSeen) > within: + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale, + Kind: kindBackupStale, Machine: r.Machine, Severity: severity, + Summary: fmt.Sprintf("%s of %s on %s is %s and has no good backup on record, %s after it was first "+ + "declared — is node-backup held there, and do its nights succeed? (`node-backup.backed-up`)", + r.Item, r.Module, r.Machine, class, now.Sub(r.FirstSeen).Round(time.Hour))}) + case r.LastBackup != nil && now.Sub(*r.LastBackup) > within: + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale, + Kind: kindBackupStale, Machine: r.Machine, Severity: severity, + Summary: fmt.Sprintf("%s of %s on %s is %s and its newest good backup is from %s, older than its bound "+ + "of %s", r.Item, r.Module, r.Machine, class, r.LastBackup.UTC().Format(time.RFC3339), within)}) + } + } + } + for _, k := range keysSorted(shrunkDatasets) { + ds := shrunkDatasets[k] + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, + ID: ds.machine + ".dataset." + strings.ReplaceAll(ds.dataset, "/", "-"), Token: kindDataShrank, + Kind: kindDataShrank, Machine: ds.machine, Severity: severityOf(ds.class), Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("the dataset %s on %s shrank to %s from %s within %d days — more than half of what it held "+ + "is gone; it holds %s", ds.dataset, ds.machine, sizeWords(&ds.size), sizeWords(&ds.peak), + int(shrinkWindow.Hours()/24), strings.Join(ds.items, ", "))}) + } + // The redundant storage watched data is on: one condition per array, as loud as the most precious + // item on it — the array, not each item, is what degrades. + for _, where := range keysSorted(arrays) { + rs := arrays[where] + red := rs[0].Redundancy + if red.Healthy != nil && *red.Healthy { + continue + } + class, machine := catalogue.ClassValuable, rs[0].Machine + var names []string + for _, r := range rs { + class = catalogue.StricterClass(class, r.Class) + names = append(names, r.Module+"/"+r.Item) + } + state := "could not be read" + if red.Healthy != nil { + state = "is NOT healthy" + } + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, + ID: machine + ".array." + strings.NewReplacer("/", "-", ":", "-").Replace(red.Kind+"-"+red.Where), + Token: kindArrayDegraded, Kind: kindArrayDegraded, Machine: machine, Severity: severityOf(class), + Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("the %s storage %s on %s %s: %s — and it is what protects %s", red.Kind, red.Where, machine, + state, firstLine(red.Said), strings.Join(names, ", "))}) + } + // The same item on several machines: a copy that is in use and far smaller than one kept elsewhere is + // an empty replacement. Only against a retired copy — a module running on two machines on purpose + // keeps two different sets of data. + for _, key := range keysSorted(byItem) { + rs := byItem[key] + for _, a := range rs { + if a.Retired() || a.Size == nil || !catalogue.Watched(a.Class) || !inventory.Comparable(a.Precision) { + continue + } + for _, o := range rs { + if o.Machine == a.Machine || !o.Retired() || o.Size == nil || !inventory.Comparable(o.Precision) || + !replacedByLess(*a.Size, *o.Size) { + continue + } + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, + ID: keyOfItem(a.Machine, a.Module, a.Item), Token: kindEmptyReplacement, Kind: kindEmptyReplacement, + Machine: a.Machine, Also: []string{o.Machine}, Severity: severityOf(a.Class), Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s of %s on %s holds %s, and the copy %s kept on %s holds %s: %s is running on "+ + "an empty replacement of its data. Move the data, or assign it back where its data is", + a.Item, a.Module, a.Machine, sizeWords(a.Size), o.Module, o.Machine, sizeWords(o.Size), a.Module)}) + break + } + } + } + out = append(out, consumerFindings(copies)...) + return out +} + +// replacedByLess is whether a copy in use is an empty replacement of a copy kept elsewhere: less than +// half of it, and at least shrinkFloor less. +func replacedByLess(inUse, kept int64) bool { + return inUse*2 < kept && kept-inUse >= shrinkFloor +} + +// consumerFindings is the same question of consumers' data at providers: one consumer, the same +// provider module on two machines. +func consumerFindings(copies []consumerCopy) []conditions.Observation { + by := map[string][]consumerCopy{} + for _, c := range copies { + k := c.Module + "/" + c.Consumer + by[k] = append(by[k], c) + } + var out []conditions.Observation + for _, k := range keysSorted(by) { + cs := by[k] + if len(cs) < 2 { + continue + } + found := false + for _, a := range cs { + if a.Retired || a.Size == nil { + continue + } + for _, o := range cs { + if o.Node == a.Node || o.Size == nil || !replacedByLess(*a.Size, *o.Size) { + continue + } + state := "active" + if o.Retired { + state = "retired" + } + out = append(out, conditions.Observation{Scope: conditions.ScopeProvider, + ID: a.Module + "." + a.Node + "." + a.Consumer, Token: kindEmptyReplacement, Kind: kindEmptyReplacement, + Machine: a.Node, Also: []string{o.Node}, Severity: severityOf(catalogue.StricterClass(a.Class, o.Class)), + Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s's data at %s on %s holds %s, and its %s copy at %s on %s holds %s: the "+ + "consumer is using an empty replacement of its data (issue 273's shape). Pin it back to %s, or move "+ + "the data first", a.Consumer, a.Module, a.Node, sizeWords(a.Size), state, o.Module, o.Node, + sizeWords(o.Size), o.Node)}) + found = true + break + } + if found { + break + } + } + if found { + continue + } + var active []consumerCopy + for _, c := range cs { + if !c.Retired { + active = append(active, c) + } + } + if len(active) >= 2 { + var where []string + var also []string + for _, c := range active { + where = append(where, c.Node+" ("+sizeWords(c.Size)+")") + also = append(also, c.Node) + } + out = append(out, conditions.Observation{Scope: conditions.ScopeProvider, + ID: active[0].Module + "." + active[0].Consumer, Token: kindDataHeldTwice, Kind: kindDataHeldTwice, + Machine: active[0].Node, Also: also[1:], Severity: conditions.Warning, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s has active data at %s on %d machines — %s — and only one is the one it uses", + active[0].Consumer, active[0].Module, len(active), strings.Join(where, ", "))}) + } + } + return out +} + +func keysSorted[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// ---- the `data` verb --------------------------------------------------------------------------- + +const dataUsage = "data [--json] [--machine ] [--retired]" + +// dataRow is one item as `data` lists it. +type dataRow struct { + Machine string `json:"machine"` + Module string `json:"module"` + Item string `json:"item"` + Class string `json:"class"` + Path string `json:"path,omitempty"` + Protection string `json:"protection,omitempty"` + // Array is the redundant storage it is on and its state, where its holder could tell. + Array string `json:"array,omitempty"` + Unmeasured string `json:"unmeasured,omitempty"` + // Precision says what the size is: exact, a dataset's whole size, partial, or none. + Precision string `json:"precision,omitempty"` + SizeBytes *int64 `json:"size-bytes,omitempty"` + LastWrite string `json:"last-write,omitempty"` + MeasuredAt string `json:"measured-at,omitempty"` + LastBackup string `json:"last-backup,omitempty"` + BackupDue string `json:"backup-within,omitempty"` + Retired string `json:"retired,omitempty"` + RetiredWhy string `json:"retired-why,omitempty"` + Deleted string `json:"deleted,omitempty"` +} + +// dataCommand is `data`: every item every machine declares, or held retired, as the self-check last +// found it. +func dataCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("data", flag.ContinueOnError) + asJSON := set.Bool("json", false, "as data") + only := set.String("machine", "", "one machine") + retiredOnly := set.Bool("retired", false, "only what is retired") + if rest, err := parseAround(set, args); err != nil { + return err + } else if len(rest) > 0 { + return errors.New(dataUsage) + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + records, err := open.inventory.Data(ctx) + if err != nil { + return err + } + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + return err + } + rows := dataRows(records, shelf, *only, *retiredOnly) + if *asJSON { + return printJSON(map[string]any{"data": rows}) + } + if len(rows) == 0 { + fmt.Println("no data on record: the self-check (D13) records what each machine declares on its next run") + return nil + } + for _, r := range rows { + state := "" + switch { + case r.Deleted != "": + state = " DELETED " + r.Deleted + case r.Retired != "": + state = " RETIRED " + r.Retired + " — " + r.RetiredWhy + } + fmt.Printf("%s %s/%s %s %s %s protected by %s%s\n", r.Machine, r.Module, r.Item, r.Class, + sizeWords(r.SizeBytes), orUnknownPath(r.Path), orNothingWord(r.Protection), state) + if r.Array != "" { + fmt.Printf(" on %s\n", r.Array) + } + if r.Precision != "" && r.Precision != "exact" { + fmt.Printf(" size: %s\n", r.Precision) + } + if r.Unmeasured != "" { + fmt.Printf(" not measured: %s\n", r.Unmeasured) + } + if r.Class == catalogue.ClassCache { + continue + } + fmt.Printf(" last write %s, measured %s, last backup %s%s\n", orNever(r.LastWrite), orNever(r.MeasuredAt), + orNever(r.LastBackup), within(r.BackupDue)) + } + return nil +} + +func dataRows(records []inventory.DataRecord, shelf map[string]catalogue.Manifest, only string, retiredOnly bool) []dataRow { + rows := []dataRow{} + stamp := func(t *time.Time) string { + if t == nil { + return "" + } + return t.UTC().Format(time.RFC3339) + } + for _, r := range records { + if only != "" && r.Machine != only { + continue + } + if retiredOnly && !r.Retired() { + continue + } + row := dataRow{Machine: r.Machine, Module: r.Module, Item: r.Item, Class: r.Class, Path: r.Path, + Protection: r.Protection, Unmeasured: r.MeasureError, Precision: r.Precision, SizeBytes: r.Size, LastWrite: stamp(r.LastWrite), MeasuredAt: stamp(r.MeasuredAt), + LastBackup: stamp(r.LastBackup), Retired: stamp(r.RetiredAt), RetiredWhy: r.RetiredWhy, + Deleted: stamp(r.DeletedAt)} + if it, ok := shelf[r.Module].DataItem(r.Item); ok && it.BackedUp() { + row.BackupDue = it.BackupWithin().String() + } + if red := r.Redundancy; red != nil { + state := "state unread" + if red.Healthy != nil && *red.Healthy { + state = "healthy" + } else if red.Healthy != nil { + state = "NOT HEALTHY" + } + row.Array = red.Kind + " " + red.Where + ", " + state + } + rows = append(rows, row) + } + return rows +} + +func orNothingWord(s string) string { + if s == "" || s == "none" { + return "nothing" + } + return s +} + +func orNever(s string) string { + if s == "" { + return "never" + } + return s +} + +func within(s string) string { + if s == "" { + return " (not backed up)" + } + return " (bound " + s + ")" +} + +// ---- cleanup of retired own data --------------------------------------------------------------- + +// The tools a node-backup holder serves to delete one retired item (novox/hq ADR 0233): the first +// takes a last restore point of it, tagged as retired, and only then removes it — in the background, +// because a large item outlasts any call — and the second says how that went. Module tools, not seat +// verbs: only the controller's `cleanup delete` calls them, as it calls a provider's provisioner_delete. +const ( + ToolDeleteRetired = "backup_delete_retired" + ToolDeletedOutcome = "backup_deleted" +) + +// deletionWait is how long `cleanup delete` follows a deletion before handing it back to the person. +var deletionWait = 8 * time.Minute + +// deletionPoll is how often it asks. +var deletionPoll = 5 * time.Second + +// deletion is a holder's account of one deletion. +type deletion struct { + Started bool `json:"started"` + Running bool `json:"running"` + Done bool `json:"done"` + OK bool `json:"ok"` + Snapshot string `json:"snapshot"` + Error string `json:"error"` +} + +// retiredData is every retired item on record, as `cleanup list` shows them. +func retiredData(records []inventory.DataRecord, now time.Time) []retiredRow { + var out []retiredRow + for _, r := range records { + if !r.Retired() { + continue + } + out = append(out, retiredRow{Node: r.Machine, Module: r.Module, Consumer: r.Item, Kind: retiredDataKind, + RetiredAt: r.RetiredAt.UTC().Format(time.RFC3339), AgeDays: int(now.Sub(*r.RetiredAt).Hours() / 24), + SizeBytes: r.Size, Why: r.RetiredWhy, Path: r.Path, Class: r.Class}) + } + return out +} + +// retiredDataKind is what `cleanup list` calls a module's own retired data, beside a provider's consumer. +const retiredDataKind = "own-data" + +// holderOn is the module holding node-backup on a machine. +func holderOn(ctx context.Context, inv *inventory.Inventory, machine string) (string, error) { + held, err := inv.Holdings(ctx) + if err != nil { + return "", err + } + for _, h := range held { + if s, known := catalogue.SeatNamed(h.Claim); known && s.Name == catalogue.BackupSeat && h.Node == machine { + return h.Module, nil + } + } + return "", fmt.Errorf("nothing holds %s on %s, and it is the backup holder that deletes retired data there "+ + "(after a last restore point)", catalogue.BackupSeat, machine) +} + +// deleteRetiredData has a machine's backup holder delete one retired item: never one declared now, and +// never one not retired. The holder takes a last restore point of it first, so the deletion can be +// undone until a person forgets that restore point; the record says deleted only once the holder says +// it is. +func deleteRetiredData(ctx context.Context, conn *nats.Conn, open *stores, r inventory.DataRecord, f handActFlags) error { + inv := open.inventory + if !r.Retired() { + return fmt.Errorf("%s of %s on %s is not retired — only retired data is deleted. Nothing was done", + r.Item, r.Module, r.Machine) + } + if r.Path == "" { + return fmt.Errorf("%s of %s on %s was never measured, so where it is was never said; nothing was deleted", + r.Item, r.Module, r.Machine) + } + if plan, _, err := planFor(ctx, open, r.Machine); err == nil { + for _, m := range plan.Modules { + if _, still := m.DataItem(r.Item); still && m.Module == r.Module { + return fmt.Errorf("%s runs on %s again and declares %s: it is not retired any more. Nothing was done", + r.Module, r.Machine, r.Item) + } + } + } + holder, err := holderOn(ctx, inv, r.Machine) + if err != nil { + return err + } + f.record(ctx, "cleanup delete", []string{r.Machine, r.Module, r.Item}) + args := map[string]any{"module": r.Module, "item": r.Item, "path": r.Path, "confirm": r.Item, + "why": strings.TrimSpace(*f.why), "by": link.Caller(), "via": link.ViaController} + ask := func(tool string) (deletion, error) { + var d deletion + answer, err := link.AskModuleToolOn(ctx, conn, holder, tool, r.Machine, args, 25*time.Second) + if err != nil { + return d, err + } + if answer.Error != "" { + return d, fmt.Errorf("%s on %s refused: %s", holder, r.Machine, answer.Error) + } + return d, unmarshalAnswer(answer, &d) + } + d, err := ask(ToolDeleteRetired) + if err != nil { + return err + } + for waited := time.Duration(0); !d.Done && waited < deletionWait; waited += deletionPoll { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(deletionPoll): + } + if d, err = ask(ToolDeletedOutcome); err != nil { + return err + } + } + switch { + case !d.Done: + fmt.Printf("%s on %s is still taking the last restore point of %s and deleting it; `cleanup list` keeps "+ + "showing it until the holder says it is done — the same `cleanup delete` again reads how it went\n", + holder, r.Machine, r.Path) + return nil + case !d.OK: + return fmt.Errorf("%s on %s did NOT delete %s: %s", holder, r.Machine, r.Path, d.Error) + } + if err := inv.MarkDataDeleted(ctx, r.Machine, r.Module, r.Item, link.Caller(), strings.TrimSpace(*f.why), time.Now()); err != nil { + return fmt.Errorf("%s deleted %s on %s, and it could not be recorded: %w", holder, r.Path, r.Machine, err) + } + fmt.Printf("%s on %s deleted %s of %s (%s, %s); its last restore point is %s, kept until a person forgets it\n", + holder, r.Machine, r.Item, r.Module, r.Path, sizeWords(r.Size), orNever(d.Snapshot)) + return nil +} + +// keptOnUnassign says, for an unassignment, the irreplaceable and valuable data each module leaves in its +// own directories on the machine: +// kept, and retired at the self-check's next run. +func keptOnUnassign(ctx context.Context, inv *inventory.Inventory, machine string, modules []string) []string { + records, err := inv.Data(ctx) + if err != nil { + return []string{"what it leaves behind could not be read from the mesh's record: " + err.Error()} + } + var out []string + for _, r := range records { + if r.Machine != machine || r.DeletedAt != nil || !catalogue.Retires(r.Class) || !r.Owned { + continue + } + for _, m := range modules { + if r.Module == m { + out = append(out, fmt.Sprintf("%s's %s (%s, %s) stays where it is: it is %s, so it is retired, "+ + "never removed — `cleanup list` shows it, `cleanup delete` alone removes it (novox/hq ADR 0233)", + r.Module, r.Item, orUnknownPath(r.Path), sizeWords(r.Size), r.Class)) + } + } + } + return out +} diff --git a/cmd/mesh-controller/data_test.go b/cmd/mesh-controller/data_test.go new file mode 100644 index 0000000..2ed87e2 --- /dev/null +++ b/cmd/mesh-controller/data_test.go @@ -0,0 +1,460 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/link" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +func bytesOf(n int64) *int64 { return &n } + +func when(t time.Time) *time.Time { return &t } + +func shelfFor(t *testing.T, manifests ...string) map[string]catalogue.Manifest { + t.Helper() + out := map[string]catalogue.Manifest{} + for _, raw := range manifests { + m, err := catalogue.ParseManifest([]byte(raw)) + if err != nil { + t.Fatal(err) + } + out[m.Module] = m + } + return out +} + +const houseManifest = `{"module":"house","version":"1", + "data":{"own":[{"id":"config","path":"${dir:config}","class":"irreplaceable","active":"1d"}]}, + "resources":[{"id":"config","type":"directory","mode":"0700"}]}` + +func findingsByKind(obs []conditions.Observation) map[string]conditions.Observation { + out := map[string]conditions.Observation{} + for _, o := range obs { + out[o.Kind] = o + } + return out +} + +// THE INCIDENT (issue 273), replayed against what D13 reads: five applications on the home server bound, +// by one changed rule, to the store on the control node, which made each an empty database — while +// their real databases, hundreds of megabytes each, sat on the home server's own store, by then retired +// because the mesh no longer asked for them there. Each is an empty replacement, naming both machines +// and the pin back: a warning for the store's consumers, whose data is valuable; urgent for one that says +// its data there is irreplaceable (`kept-by`). +func TestAnEmptyReplacementOfAConsumersDataIsSaid(t *testing.T) { + var copies []consumerCopy + for _, app := range []string{"mesh_home_board", "mesh_home_flows", "mesh_home_agents", "mesh_home_game", "mesh_home_cars"} { + copies = append(copies, + consumerCopy{Node: "home", Module: "postgres", Consumer: app, Size: bytesOf(400 << 20), Retired: true, Class: "valuable"}, + consumerCopy{Node: "anchor", Module: "postgres", Consumer: app, Size: bytesOf(9 << 20), Class: "valuable"}) + } + copies[1].Class = "irreplaceable" // the photo site's own database says so + got := dataFindings(nil, nil, nil, copies, nil, time.Now()) + if len(got) != 5 { + t.Fatalf("%d findings for five empty replacements: %+v", len(got), got) + } + for i, o := range got { + want := conditions.Warning + if strings.Contains(o.ID, "mesh_home_board") { + want = conditions.Urgent + } + _ = i + if o.Kind != kindEmptyReplacement || o.Severity != want || o.Machine != "anchor" || + len(o.Also) != 1 || o.Also[0] != "home" || !strings.Contains(o.Summary, "Pin it back to home") { + t.Errorf("%+v", o) + } + } + + // While the old copy is still active (the first ten minutes), it is the same finding. + copies[0].Retired = false + copies[1].Class = "valuable" + if got := dataFindings(nil, nil, nil, copies[:2], nil, time.Now()); len(got) != 1 || got[0].Kind != kindEmptyReplacement { + t.Fatalf("with the old copy still active: %+v", got) + } +} + +// A move a person made — the data moved first, then pinned — leaves a full copy at the new provider and +// a retired one at the old: nothing to say here; `cleanup` covers the old one. +func TestADeliberateMoveIsNoEmptyReplacement(t *testing.T) { + copies := []consumerCopy{ + {Node: "home", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(400 << 20), Retired: true}, + {Node: "anchor", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(402 << 20)}, + } + if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 { + t.Fatalf("a deliberate move raised %+v", got) + } + // Two small databases differing by less than the floor are not a finding either. + copies[0].Size, copies[1].Size = bytesOf(12<<20), bytesOf(8<<20) + if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 { + t.Fatalf("noise between two empty databases raised %+v", got) + } +} + +// Where a provider cannot say sizes, a consumer active at two providers is still said — as a warning, +// since which one is empty cannot be told. +func TestConsumerDataActiveTwiceWithoutSizesIsAWarning(t *testing.T) { + copies := []consumerCopy{ + {Node: "home", Module: "minio", Consumer: "mesh_home_photos"}, + {Node: "anchor", Module: "minio", Consumer: "mesh_home_photos"}, + } + got := dataFindings(nil, nil, nil, copies, nil, time.Now()) + if len(got) != 1 || got[0].Kind != kindDataHeldTwice || got[0].Severity != conditions.Warning { + t.Fatalf("%+v", got) + } +} + +// The same incident for a module's own data: a module unassigned from one machine and assigned on +// another starts over in an empty directory while its full one is kept, retired, where it was. +func TestAnEmptyReplacementOfAModulesOwnDataIsUrgent(t *testing.T) { + now := time.Now() + retired := now.Add(-time.Hour) + records := []inventory.DataRecord{ + {Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config", + Size: bytesOf(2 << 30), RetiredAt: &retired, FirstSeen: now.Add(-90 * 24 * time.Hour)}, + {Machine: "anchor", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config", + Size: bytesOf(1 << 20), FirstSeen: now.Add(-time.Hour), LastWrite: when(now)}, + } + got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now)) + o, ok := got[kindEmptyReplacement] + if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "home" { + t.Fatalf("%+v", got) + } + // The same of a valuable item is a warning. + records[0].Class, records[1].Class = "valuable", "valuable" + if o := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))[kindEmptyReplacement]; o.Severity != conditions.Warning { + t.Fatalf("a valuable empty replacement: %+v", o) + } + records[0].Class, records[1].Class = "irreplaceable", "irreplaceable" + // Two machines running a module on purpose, both active, keep two sets of data: nothing to say. + records[0].RetiredAt = nil + if got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now)); got[kindEmptyReplacement].Kind != "" { + t.Fatalf("two active copies were read as a replacement: %+v", got) + } +} + +// An irreplaceable item that lost more than half of its largest size in a week is urgent; a smaller loss, +// or a loss under the floor, is not a finding. +func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) { + now := time.Now() + r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", + Size: bytesOf(300 << 20), FirstSeen: now.Add(-30 * 24 * time.Hour), LastWrite: when(now), LastBackup: when(now)} + shelf := shelfFor(t, houseManifest) + o := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): 1 << 30}, shelf, nil, nil, now))[kindDataShrank] + if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "shrank") { + t.Fatalf("%+v", o) + } + for _, peak := range []int64{500 << 20, 20 << 20} { + if got := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): peak}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" { + t.Errorf("a peak of %d raised a shrink", peak) + } + } + small := r + small.Size = bytesOf(1 << 20) + if got := findingsByKind(dataFindings([]inventory.DataRecord{small}, map[string]int64{r.Key(): 10 << 20}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" { + t.Error("a loss under the floor raised a shrink") + } +} + +// Data said to be written all the time and not written; data with no backup or an old one — urgent when +// irreplaceable, a warning when valuable; and a new item given its bound before it is said. +func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) { + now := time.Now() + shelf := shelfFor(t, houseManifest) + r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", + Size: bytesOf(1 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), LastWrite: when(now.Add(-3 * 24 * time.Hour)), + LastBackup: when(now.Add(-72 * time.Hour))} + got := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now)) + if got[kindDataQuiet].Severity != conditions.Urgent || got[kindBackupStale].Severity != conditions.Urgent { + t.Fatalf("irreplaceable: %+v", got) + } + valuable := r + valuable.Class, valuable.MeasuredAt = "valuable", when(now) // measured: a holder is there to take its backup + if got := findingsByKind(dataFindings([]inventory.DataRecord{valuable}, nil, shelf, nil, nil, now)); got[kindDataQuiet].Severity != conditions.Warning || + got[kindBackupStale].Severity != conditions.Warning { + t.Fatalf("valuable: %+v", got) + } + never := r + never.LastBackup = nil + if o := findingsByKind(dataFindings([]inventory.DataRecord{never}, nil, shelf, nil, nil, now))[kindBackupStale]; !strings.Contains(o.Summary, "no good backup") { + t.Fatalf("never backed up: %+v", o) + } + fresh := never + fresh.FirstSeen, fresh.LastWrite = now.Add(-time.Hour), when(now) + if got := dataFindings([]inventory.DataRecord{fresh}, nil, shelf, nil, nil, now); len(got) != 0 { + t.Fatalf("an item declared an hour ago, before its first night, raised %+v", got) + } +} + +// An item retired more than thirty days waits for a person; less, it is only listed. +func TestRetiredDataWaitingThirtyDaysIsSaid(t *testing.T) { + now := time.Now() + old, recent := now.Add(-31*24*time.Hour), now.Add(-2*24*time.Hour) + records := []inventory.DataRecord{ + {Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &old}, + {Machine: "home", Module: "attic", Item: "boxes", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &recent}, + } + got := dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now) + if len(got) != 1 || got[0].Kind != kindCleanupWaiting || !strings.Contains(got[0].Summary, "cleanup delete home house config") { + t.Fatalf("%+v", got) + } + if rows := retiredData(records, now); len(rows) != 2 || rows[0].Kind != retiredDataKind { + t.Fatalf("cleanup list: %+v", rows) + } +} + +// The holder's answer reads into measurements, by module and item. +func TestTheHoldersAnswerIsRead(t *testing.T) { + raw := []byte(`[{"module":"postgres","runs":1,"paths":["/var/lib/mesh-store/dumps"],"lastNight":null,"restorePoints":3, + "data":[{"item":"store","class":"irreplaceable","path":"/var/lib/mesh-store","covered_by":"/var/lib/mesh-store/dumps", + "size_bytes":1073741824,"last_write":"2026-10-06T10:00:00Z","measured_at":"2026-10-06T10:05:00Z","last_backup":"2026-10-06T03:10:00Z"}]}]`) + got, err := readHolder(raw) + if err != nil { + t.Fatal(err) + } + m := got["postgres"]["store"] + if m.Path != "/var/lib/mesh-store" || m.Size == nil || *m.Size != 1<<30 || m.LastBackup == nil || m.MeasuredAt == nil { + t.Fatalf("%+v", m) + } + // An older holder, which says no data, reads as nothing measured rather than a failure. + if got, err := readHolder([]byte(`[{"module":"postgres","runs":1,"paths":[]}]`)); err != nil || len(got) != 0 { + t.Fatalf("%v, %v", got, err) + } +} + +// fakeHolder answers node-backup's `backed-up` on one machine over a real bus, with what it is told it +// measured. +type fakeHolder struct { + mu sync.Mutex + modules []map[string]any +} + +func (f *fakeHolder) set(modules ...map[string]any) { + f.mu.Lock() + defer f.mu.Unlock() + f.modules = modules +} + +func (f *fakeHolder) serve(t *testing.T, conn *nats.Conn, node string) { + t.Helper() + sub, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) { + f.mu.Lock() + defer f.mu.Unlock() + body, _ := json.Marshal(map[string]any{"result": f.modules, "error": "", "node": node}) + _ = m.Respond(body) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + if err := conn.Flush(); err != nil { + t.Fatal(err) + } +} + +func measuredHouse(path string, size int64, at time.Time) map[string]any { + return map[string]any{"module": "house", "runs": 0, "paths": []string{path}, "data": []map[string]any{{ + "item": "config", "class": "irreplaceable", "path": path, "covered_by": path, "size_bytes": size, + "last_write": at, "measured_at": at, "last_backup": at}}} +} + +// UNASSIGNING A MODULE WITH IRREPLACEABLE DATA KEEPS THE DATA, and assigning it elsewhere onto an empty +// directory is an empty replacement — through the real stores and a real bus. The unassignment says the +// data stays; the self-check's next run retires it (kept, listed by cleanup), and when the module comes +// up on another machine with an empty directory while the full one waits retired, that is urgent. +func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { + t.Fatal(err) + } + register(t, open, catalogue.Manifest{Module: "keeper", Version: "1", + Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}) + house, err := catalogue.ParseManifest([]byte(houseManifest)) + if err != nil { + t.Fatal(err) + } + register(t, open, house) + if _, err := assign(ctx, open, "laptop", "house"); err == nil { + t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up") + } + for _, node := range []string{"laptop", "anchor"} { + if _, err := assign(ctx, open, node, "keeper"); err != nil { + t.Fatal(err) + } + } + if _, err := assign(ctx, open, "laptop", "house"); err != nil { + t.Fatal(err) + } + + conn := onATestBus(t) + js, err := broker.Dial(os.Getenv("MESH_TEST_NATS")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + laptop, anchor := &fakeHolder{}, &fakeHolder{} + laptop.serve(t, conn, "laptop") + anchor.serve(t, conn, "anchor") + now := time.Now() + heard := &watchdogs{last: &signalFacts{now: now, machines: []machineFacts{ + {name: "laptop", lastHeard: now}, {name: "anchor", lastHeard: now}}}} + d := &doctor{open: open, js: js, watchdogs: heard} + var d13 probe + for _, p := range probeRegistry { + if p.ID == probeDataID { + d13 = p + } + } + run := func() []conditions.Observation { + t.Helper() + probing := context.WithValue(ctx, probeAsksKey{}, d13) + obs, err := probeData(probing, d) + if err != nil { + t.Fatal(err) + } + return obs + } + + laptop.set(measuredHouse("/var/lib/house/config", 2<<30, now)) + if obs := run(); len(obs) != 0 { + t.Fatalf("a measured, backed-up item raised %+v", obs) + } + r, err := inv.DataOf(ctx, "laptop", "house", "config") + if err != nil || r.Path != "/var/lib/house/config" || r.Size == nil || *r.Size != 2<<30 || r.LastBackup == nil { + t.Fatalf("what the holder measured was not kept: %+v, %v", r, err) + } + + said, err := unassign(ctx, open, "laptop", "house") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(said, "house's config (/var/lib/house/config, 2.0 GB) stays where it is") { + t.Fatalf("the unassignment does not say the data stays:\n%s", said) + } + laptop.set() + run() + r, err = inv.DataOf(ctx, "laptop", "house", "config") + if err != nil || !r.Retired() || r.Path != "/var/lib/house/config" { + t.Fatalf("unassigned, the irreplaceable item is not kept retired: %+v, %v", r, err) + } + records, _ := inv.Data(ctx) + if rows := retiredData(records, time.Now()); len(rows) != 1 || rows[0].Path != "/var/lib/house/config" { + t.Fatalf("cleanup list: %+v", rows) + } + + // Assigned on the anchor, onto an empty directory. + if _, err := assign(ctx, open, "anchor", "house"); err != nil { + t.Fatal(err) + } + anchor.set(measuredHouse("/var/lib/house/config", 300<<10, time.Now())) + obs := findingsByKind(run()) + o, ok := obs[kindEmptyReplacement] + if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "laptop" { + t.Fatalf("an empty replacement of a module's data was not urgent: %+v", obs) + } +} + +// Data on redundant storage: the array it is on is watched, one condition per array as loud as the most +// precious item on it; an item said to be on redundancy and found on plain storage is said; an item +// whose path is gone is said. +func TestTheArrayUnderDataIsWatched(t *testing.T) { + now := time.Now() + media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}], + "data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array, no room to copy"}, + {"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array, no room to copy"}]}}` + shelf := shelfFor(t, media) + sick := false + on := func(item string, healthy *bool) inventory.DataRecord { + return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable", Owned: false, + Path: "/tank/" + item, Size: bytesOf(40 << 40), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now), + Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: healthy, Said: "pool 'tank' is DEGRADED"}} + } + got := dataFindings([]inventory.DataRecord{on("films", &sick), on("shows", &sick)}, nil, shelf, nil, nil, now) + if len(got) != 1 || got[0].Kind != kindArrayDegraded || got[0].Severity != conditions.Urgent || + !strings.Contains(got[0].Summary, "media/films, media/shows") { + t.Fatalf("%+v", got) + } + well := true + if got := dataFindings([]inventory.DataRecord{on("films", &well)}, nil, shelf, nil, nil, now); len(got) != 0 { + t.Fatalf("a healthy array raised %+v", got) + } + plain := on("films", nil) + plain.Redundancy = nil + if o := findingsByKind(dataFindings([]inventory.DataRecord{plain}, nil, shelf, nil, nil, now))[kindProtectionMissing]; o.Severity != conditions.Urgent { + t.Fatalf("redundancy said and not found: %+v", o) + } + gone := on("films", &well) + gone.MeasureError, gone.Size = "/tank/films does not exist", bytesOf(0) + if o := findingsByKind(dataFindings([]inventory.DataRecord{gone}, map[string]int64{gone.Key(): 40 << 40}, shelf, nil, nil, now))[kindDataMissing]; o.Severity != conditions.Urgent { + t.Fatalf("a vanished library: %+v", o) + } +} + +// What a consumer keeps with its provider as irreplaceable holds the provider's item to that class: +// the photo site's objects make the object store's data an urgent matter. +func TestKeptByHoldsTheProvidersItemToTheConsumersClass(t *testing.T) { + objects := `{"module":"objects","version":"1","provides":[{"name":"s3-bucket","scope":"mesh"}],"grants":{"s3-bucket":"${dir:g}"}, + "data":{"own":[{"id":"data","path":"${dir:data}","class":"valuable"}],"consumers":{"s3-bucket":{"class":"valuable","in":"data"}}}, + "resources":[{"id":"g","type":"directory","mode":"0700"},{"id":"data","type":"directory","mode":"0700"}]}` + photos := `{"module":"photos","version":"1","requires":["s3-bucket"],"data":{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}}` + shelf := shelfFor(t, objects, photos) + bindings := []inventory.Binding{{Machine: "anchor", Consumer: "photos", Provision: "s3-bucket", + Provider: catalogue.Chosen{Node: "anchor", Module: "objects"}}} + upgraded, keptBy := keptByClasses(bindings, shelf) + if upgraded["anchor/objects/data"] != "irreplaceable" || keptBy["objects/mesh_anchor_photos"] != "irreplaceable" { + t.Fatalf("upgraded %v, kept by %v", upgraded, keptBy) + } + now := time.Now() + r := inventory.DataRecord{Machine: "anchor", Module: "objects", Item: "data", Class: "valuable", Owned: true, + Size: bytesOf(10 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), MeasuredAt: when(now), LastBackup: when(now.Add(-72 * time.Hour))} + if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, upgraded, now))[kindBackupStale]; o.Severity != conditions.Urgent { + t.Fatalf("the photos' store without a backup: %+v", o) + } + if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now))[kindBackupStale]; o.Severity != conditions.Warning { + t.Fatalf("a valuable store without a backup: %+v", o) + } +} + +// Items measured from one dataset's counters share its size: a shrink of the dataset is one condition +// naming every item on it, not one per item; and a partial walk's lower bound is never compared. +func TestADatasetShrinksOnceAndAPartialSizeIsNeverCompared(t *testing.T) { + now := time.Now() + media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}], + "data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array","measure":"dataset"}, + {"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array","measure":"dataset"}]}}` + shelf := shelfFor(t, media, houseManifest) + well := true + on := func(item string, size int64) inventory.DataRecord { + return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable", + Size: bytesOf(size), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now), + Precision: "dataset tank/media: its whole size", + Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: &well}} + } + films, shows := on("films", 30<<40), on("shows", 30<<40) + peaks := map[string]int64{films.Key(): 90 << 40, shows.Key(): 90 << 40} + got := dataFindings([]inventory.DataRecord{films, shows}, peaks, shelf, nil, nil, now) + if len(got) != 1 || got[0].Kind != kindDataShrank || got[0].Severity != conditions.Urgent || + !strings.Contains(got[0].Summary, "media/films, media/shows") { + t.Fatalf("%+v", got) + } + partial := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", + Size: bytesOf(1 << 20), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now), LastWrite: when(now), + LastBackup: when(now), Precision: "partial: measured partially"} + if got := dataFindings([]inventory.DataRecord{partial}, map[string]int64{partial.Key(): 1 << 30}, shelf, nil, nil, now); len(got) != 0 { + t.Fatalf("a partial size was compared: %+v", got) + } +} diff --git a/cmd/mesh-controller/doctor.go b/cmd/mesh-controller/doctor.go index 310f31a..1394242 100644 --- a/cmd/mesh-controller/doctor.go +++ b/cmd/mesh-controller/doctor.go @@ -102,6 +102,14 @@ var probeRegistry = []probe{ {ID: probeBindingsID, Asserts: "every consumer of a provision that keeps its data is bound where it was last " + "sent, or moves by a pin", From: "issue 273, ADR 0232", Kind: kindBindingMoved, Raises: []string{kindBindingKept, kindBindingMoving}, Phase: 2, run: probeBindings}, + {ID: probeDataID, Asserts: "every item of data a machine declares is measured, is there, holds what it held, is " + + "written where it should be, is backed up within its bound or sits on healthy redundant storage, and is no " + + "empty replacement of a copy kept elsewhere; what a machine no longer declares that is irreplaceable or " + + "valuable is retired, not forgotten", From: "issue 273, ADR 0233", + Kind: kindDataShrank, Raises: []string{kindEmptyReplacement, kindDataHeldTwice, kindDataQuiet, kindBackupStale, + kindDataUnmeasured, kindDataMissing, kindArrayDegraded, kindProtectionMissing, kindCleanupWaiting}, + Phase: 2, run: probeData, + Asks: []broker.SeatVerb{{Seat: "node-backup", Verb: "backed-up"}}}, {ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals", From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs}, } diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index c6b7147..c63f756 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -170,6 +170,9 @@ func run() error { return retireCommand(ctx, args[1:]) case "cleanup": return cleanupCommand(ctx, args[1:]) + // The data every machine declares, as the self-check last found it (novox/hq ADR 0233). + case "data": + return dataCommand(ctx, args[1:]) case "version": fmt.Println(version) return nil diff --git a/cmd/mesh-controller/retire_verbs.go b/cmd/mesh-controller/retire_verbs.go index 30cb4e2..7b6368d 100644 --- a/cmd/mesh-controller/retire_verbs.go +++ b/cmd/mesh-controller/retire_verbs.go @@ -258,10 +258,21 @@ func cleanupCommand(ctx context.Context, args []string) error { } defer open.Close() return onTheBus(func(conn *nats.Conn) error { - return deleteOlderThan(ctx, open.inventory, conn, *olderThan, *confirm, f, time.Now()) + return deleteOlderThan(ctx, open, conn, *olderThan, *confirm, f, time.Now()) }) case *olderThan == 0 && len(rest) == 3 && !*confirm: + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() return onTheBus(func(conn *nats.Conn) error { + // A module's own retired data, when the mesh holds such an item (novox/hq ADR 0233); a + // provider's retired consumer otherwise. + if r, err := open.inventory.DataOf(ctx, rest[0], rest[1], rest[2]); err == nil && r.DeletedAt == nil && + r.RetiredAt != nil { + return deleteRetiredData(ctx, conn, open, r, f) + } return deleteRetired(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, rest[2], f) }) } @@ -285,6 +296,10 @@ type retiredRow struct { Why string `json:"why,omitempty"` // Access is "kept" for a consumer of a mark-only provider: retired on record, still reachable. Access string `json:"access,omitempty"` + // Path and Class are a module's own retired data's (Kind own-data, novox/hq ADR 0233): where it is + // kept on its machine, and its class. Consumer is then the item. + Path string `json:"path,omitempty"` + Class string `json:"class,omitempty"` } // retiredListing is every provider's retired consumers, and the providers that could not say. @@ -299,7 +314,15 @@ func gatherRetired(ctx context.Context, inv *inventory.Inventory, conn *nats.Con if err != nil { return retiredListing{}, err } - return retiredOf(ctx, conn, instances, now), nil + listing := retiredOf(ctx, conn, instances, now) + // And every module's own data retired on its machine (novox/hq ADR 0233). + records, err := inv.Data(ctx) + if err != nil { + return retiredListing{}, err + } + listing.Retired = append(listing.Retired, retiredData(records, now)...) + sort.SliceStable(listing.Retired, func(i, j int) bool { return listing.Retired[i].AgeDays > listing.Retired[j].AgeDays }) + return listing, nil } func retiredOf(ctx context.Context, conn *nats.Conn, instances []providerInstance, now time.Time) retiredListing { @@ -332,7 +355,7 @@ func printRetired(l retiredListing, asJSON bool) error { return printJSON(l) } if len(l.Retired) == 0 { - fmt.Println("no provider holds a retired consumer") + fmt.Println("no provider holds a retired consumer, and no machine holds retired data") } for _, r := range l.Retired { age := "age unknown" @@ -346,6 +369,11 @@ func printRetired(l retiredListing, asJSON bool) error { if r.Access == "kept" { kind += " [MARK ONLY: access kept until deleted]" } + if r.Kind == retiredDataKind { + fmt.Printf("%s on %s: its own %s, %s, at %s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer, + r.Class, r.Path, age, sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why)) + continue + } fmt.Printf("%s on %s: %s%s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer, kind, age, sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why)) } @@ -391,16 +419,16 @@ func deleteRetired(ctx context.Context, conn *nats.Conn, p providerInstance, con // deleteOlderThan lists every consumer retired more than days ago, and deletes them only with confirm. // One whose age the provider cannot say is never in it. -func deleteOlderThan(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, days int, confirm bool, +func deleteOlderThan(ctx context.Context, open *stores, conn *nats.Conn, days int, confirm bool, f handActFlags, now time.Time) error { - listing, err := gatherRetired(ctx, inv, conn, now) + listing, err := gatherRetired(ctx, open.inventory, conn, now) if err != nil { return err } - return deleteFrom(ctx, conn, listing, days, confirm, f) + return deleteFrom(ctx, conn, open, listing, days, confirm, f) } -func deleteFrom(ctx context.Context, conn *nats.Conn, listing retiredListing, days int, confirm bool, f handActFlags) error { +func deleteFrom(ctx context.Context, conn *nats.Conn, open *stores, listing retiredListing, days int, confirm bool, f handActFlags) error { var due []retiredRow unknown := 0 for _, r := range listing.Retired { @@ -431,7 +459,16 @@ func deleteFrom(ctx context.Context, conn *nats.Conn, listing retiredListing, da } var failed []string for _, r := range due { - if err := deleteRetired(ctx, conn, providerInstance{Node: r.Node, Module: r.Module}, r.Consumer, f); err != nil { + var err error + if r.Kind == retiredDataKind { + var rec inventory.DataRecord + if rec, err = open.inventory.DataOf(ctx, r.Node, r.Module, r.Consumer); err == nil { + err = deleteRetiredData(ctx, conn, open, rec, f) + } + } else { + err = deleteRetired(ctx, conn, providerInstance{Node: r.Node, Module: r.Module}, r.Consumer, f) + } + if err != nil { failed = append(failed, err.Error()) } } diff --git a/cmd/mesh-controller/retirement_test.go b/cmd/mesh-controller/retirement_test.go index 2784f09..e43912b 100644 --- a/cmd/mesh-controller/retirement_test.go +++ b/cmd/mesh-controller/retirement_test.go @@ -380,12 +380,12 @@ func TestNatsCleanupDeletesOnlyTheNamedRetiredConsumer(t *testing.T) { t.Fatalf("%+v", listing) } fake.deleted = nil - said = printed(t, func() error { return deleteFrom(t.Context(), conn, listing, 30, false, whyFlags(t, "tidy")) }) + said = printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, false, whyFlags(t, "tidy")) }) if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") || strings.Contains(said, "young") { t.Fatalf("deleted %v; said %s", fake.deleted, said) } - printed(t, func() error { return deleteFrom(t.Context(), conn, listing, 30, true, whyFlags(t, "tidy")) }) + printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, true, whyFlags(t, "tidy")) }) if !slices.Equal(fake.deleted, []string{"old"}) { t.Fatalf("confirmed, deleted %v", fake.deleted) } diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 9813068..a802c43 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -499,6 +499,15 @@ func (a *verbArguments) commandLine() ([]string, error) { return argv, nil } return []string{"cleanup", "list", "--json"}, nil + case "data": + argv := []string{"data", "--json"} + if m := str("machine"); m != "" { + argv = append(argv, "--machine", m) + } + if on("retired") { + argv = append(argv, "--retired") + } + return argv, nil case "doctor": which := 0 argv := []string{"doctor"} @@ -598,7 +607,7 @@ func (a *verbArguments) commandLine() ([]string, error) { // jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well. var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true, - "hand-acts": true, "durations": true, "conditions": true, "doctor": true, "retire": true, "cleanup": true} + "hand-acts": true, "durations": true, "conditions": true, "doctor": true, "retire": true, "cleanup": true, "data": true} // repairingCommand names a command line that repairs by hand, and so says why: a push, a plan stopped // or closed, a consumer re-made (novox/hq to-be 45 §7). Empty for any other. diff --git a/cmd/mesh-controller/seatverbs_schema_test.go b/cmd/mesh-controller/seatverbs_schema_test.go index 7363ebb..35656dc 100644 --- a/cmd/mesh-controller/seatverbs_schema_test.go +++ b/cmd/mesh-controller/seatverbs_schema_test.go @@ -276,6 +276,7 @@ var accountedFlags = map[string]map[string]string{ "conditions": {"json": "set by the verb: the answer is data"}, "retire": {"json": "set by the verb: the answer is data"}, "cleanup": {"json": "set by the verb: the answer is data"}, + "data": {"json": "set by the verb: the answer is data"}, "conditions history": {"json": "set by the verb: the answer is data"}, "conditions show": {"json": "set by the verb: the answer is data"}, "healers": {"json": "set by the verb: the answer is data"}, diff --git a/internal/broker/nats.go b/internal/broker/nats.go index b72b8d7..7b0cd29 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -133,7 +133,11 @@ type SeatVerb struct{ Seat, Verb string } // to the probe registry is the reason they cannot drift** — a probe that calls a verb its grant does // not name is refused by the bus on every run (found live on 2026-10-06: D8 timed out on each // machine, refused). Asked of any machine (`.*`), read-only verbs, nothing else of the seat. -var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}} +// +// D13 reads every machine's backup holder: what it measured of the data declared there (novox/hq ADR +// 0233). +var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}, + {Seat: "node-backup", Verb: "backed-up"}} // perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the // holder's machine (novox/hq ADR 0219): `paused.`, the build agent saying whether it takes work. diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 850937b..d8934f5 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -24,7 +24,7 @@ accounts { jetstream: enabled users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] } subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] } allow_responses: { max: 1, ttl: "1m" } } } diff --git a/internal/catalogue/backup_test.go b/internal/catalogue/backup_test.go index 0b46efc..7dc65f6 100644 --- a/internal/catalogue/backup_test.go +++ b/internal/catalogue/backup_test.go @@ -5,35 +5,10 @@ import ( "testing" ) -// A store provider says how its data is backed up (novox/hq ADR 0214); a provider of something that -// holds nothing does not have to. -func TestAStoreProviderWithoutABackupIsRefusedByTheCheck(t *testing.T) { - bare, err := ParseManifest([]byte(`{"module":"pg","version":"1", - "provides":[{"name":"postgres-database","scope":"mesh"}]}`)) - if err != nil { - t.Fatalf("parsing refused it, and the rule is the check's: %v", err) - } - if problems := CheckBackup(bare); len(problems) != 1 || !strings.Contains(problems[0], "node-backup") { - t.Fatalf("a store with no backup passed the check: %v", problems) - } - backed, err := ParseManifest([]byte(`{"module":"pg","version":"1", - "provides":[{"name":"postgres-database","scope":"mesh"}], - "resources":[{"id":"dumps","type":"directory","mode":"0700"}], - "contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:dumps}"}]}`)) - if err != nil { - t.Fatal(err) - } - if problems := CheckBackup(backed); len(problems) != 0 { - t.Fatalf("a store that contributes a backup was refused: %v", problems) - } - route, _ := ParseManifest([]byte(`{"module":"r","version":"1","provides":[{"name":"route","scope":"mesh"}]}`)) - if problems := CheckBackup(route); len(problems) != 0 { - t.Fatalf("a route was asked for a backup: %v", problems) - } -} - -// A backup contribution names its module's own directories; one it does not declare is refused -// where it is written rather than reaching the holder as the literal text. +// A backup contribution written by hand still names its module's own directories; one it does not +// declare is refused where it is written rather than reaching the holder as the literal text. (The +// catalogue check refuses a hand-written backup line at all since ADR 0233; parsing still reads one, +// because a module on the shelf was written before.) func TestABackupNamingAnUndeclaredDirectoryIsRefused(t *testing.T) { _, err := ParseManifest([]byte(`{"module":"pg","version":"1", "contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:nowhere}"}]}`)) @@ -42,9 +17,9 @@ func TestABackupNamingAnUndeclaredDirectoryIsRefused(t *testing.T) { } } -// The holder receives every module's backup lines with each module's own directories filled, each -// module's under a comment naming it — and a kind written in a shell's grammar is left untouched. -func TestBackupContributionsArePlacedWithTheirModulesDirectories(t *testing.T) { +// A module on the shelf from before ADR 0233, which writes its backup lines by hand and declares no +// data, is still backed up: its lines are placed as written, each module's under a comment naming it. +func TestHandWrittenBackupLinesOfAnOlderModuleArePlaced(t *testing.T) { pg, err := ParseManifest([]byte(`{"module":"pg","version":"1", "resources":[{"id":"dumps","type":"directory","path":"/srv/pg/dumps","mode":"0700"}], "contributions":[{"seat":"node-backup","kind":"backup","content":"run pg-dump-all\npath ${dir:dumps}"}]}`)) @@ -57,7 +32,7 @@ func TestBackupContributionsArePlacedWithTheirModulesDirectories(t *testing.T) { if err != nil { t.Fatal(err) } - placed, err := seatContributions([]Manifest{pg, mail}, BackupSeat, "backup", Rendering{}) + placed, err := seatContributions([]Manifest{pg, mail}, BackupSeat, "backup", Rendering{}, nil) if err != nil { t.Fatal(err) } diff --git a/internal/catalogue/catalogue_check_test.go b/internal/catalogue/catalogue_check_test.go index 998763b..5728a38 100644 --- a/internal/catalogue/catalogue_check_test.go +++ b/internal/catalogue/catalogue_check_test.go @@ -87,15 +87,18 @@ func TestNoCatalogueManifestNamesAnInstallation(t *testing.T) { } } -// TestEveryCatalogueStoreSaysHowItIsBackedUp is ADR 0214's check over the real catalogue: a module -// providing a store contributes a backup to node-backup, so a store added is a store backed up. -func TestEveryCatalogueStoreSaysHowItIsBackedUp(t *testing.T) { +// TestEveryCatalogueModuleDeclaresItsData is ADR 0233's check over the real catalogue (it replaced ADR +// 0214's store list): every provider that grants says what it keeps for its consumers, nothing writes a +// backup line by hand, every directory a container writes is declared, and every irreplaceable item is +// backed up — so a store added is a store backed up, without a list of stores to keep in step. +func TestEveryCatalogueModuleDeclaresItsData(t *testing.T) { root := catalogueRoot(t) found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) if err != nil || len(found) == 0 { t.Fatalf("no manifests under %s: %v", root, err) } - stores := 0 + shelf := Shelf{} + granting := 0 for _, p := range found { raw, err := os.ReadFile(p) if err != nil { @@ -105,18 +108,16 @@ func TestEveryCatalogueStoreSaysHowItIsBackedUp(t *testing.T) { if err != nil { continue // TestEveryCatalogueManifestParses says why } - for _, o := range m.Provides { - if storeProvisions[o.Name] { - stores++ - break - } - } - for _, problem := range CheckBackup(m) { - t.Error(problem) + if len(m.Grants) > 0 { + granting++ } + shelf[m.Module] = m } - if stores == 0 { - t.Fatal("no module in the catalogue provides a store, so this proved nothing") + for _, problem := range DataProblems(shelf) { + t.Error(problem) + } + if granting == 0 { + t.Fatal("no module in the catalogue grants a provision, so this proved nothing") } } diff --git a/internal/catalogue/data.go b/internal/catalogue/data.go new file mode 100644 index 0000000..e44ac75 --- /dev/null +++ b/internal/catalogue/data.go @@ -0,0 +1,791 @@ +package catalogue + +import ( + "bytes" + "encoding/json" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +// A module declares the data it holds, and the mesh protects and watches it from that declaration +// (novox/hq ADR 0233). +// +// **One section, `data`, for every kind of data a module keeps:** its own, by directory — a store's +// files, a mail spool, an application's uploads — or by an operator's path it was given; what it keeps +// for its consumers, by the provision they reach it through; and what of its own lives with a +// provider, by the provision it requires. Each entry has a class — how precious it is — and, for its +// own data, how it is protected; everything the mesh does is derived from those, never written per +// module: +// +// - a binding to a consumer's data does not move (ADR 0232) where the provision's class keeps data; +// - the backup holder's lines are composed from it — a module no longer writes them by hand; +// - what an unassignment leaves behind of an irreplaceable or valuable item is retired, listed by +// `cleanup list` and deleted only by `cleanup delete` (ADR 0230); +// - the self-check measures every item and says when one shrinks, disappears, stops being written, +// goes without its backup, sits on a degraded array, or is replaced by an empty copy of itself — +// urgent for what is irreplaceable, a warning for what is valuable. + +// The classes: how precious the data is. A fixed vocabulary, ranked by the operator (2026-10-06): a +// class is what the protections are derived from, so a new one is a decision, not a manifest's choice. +const ( + // ClassIrreplaceable is what must never be lost: the operator names it (the media library, the + // photo sites' storage). Protected by a backup or by a declared redundancy — one is required — + // retired rather than removed, and every alert about it is urgent. + ClassIrreplaceable = "irreplaceable" + // ClassValuable is anybody's work that would be painful to lose: in the nightly backup by default, + // retired rather than removed, and every alert about it a warning. + ClassValuable = "valuable" + // ClassRebuildable can be made again from something kept elsewhere — a clone, an index, a + // download, a night's dump — at a cost in time, not in data. In the nightly backup by default; + // forgotten when its module goes, and never alerted on. + ClassRebuildable = "rebuildable" + // ClassCache may be emptied at any moment with nothing lost but speed: never backed up, never + // measured, never alerted on. + ClassCache = "cache" + // ClassNone is a provision that keeps nothing of its consumers' (consumers only): a resolver, a + // certificate authority, an artifact store. + ClassNone = "none" +) + +// classRank orders the classes by how precious they are, so the stricter of two is chosen. +var classRank = map[string]int{ClassNone: 0, ClassCache: 1, ClassRebuildable: 2, ClassValuable: 3, ClassIrreplaceable: 4} + +// StricterClass is the more precious of two classes. +func StricterClass(a, b string) string { + if classRank[b] > classRank[a] { + return b + } + return a +} + +// Retires is whether a class's data is retired, not forgotten, when its machine no longer declares it. +func Retires(class string) bool { return class == ClassIrreplaceable || class == ClassValuable } + +// Watched is whether a class's data raises conditions: irreplaceable and valuable. +func Watched(class string) bool { return Retires(class) } + +// DefaultBackupWithin is how old the last good backup of an item may be before the self-check says so +// (novox/hq ADR 0214: a machine with data and no good backup in 48 hours). +const DefaultBackupWithin = 48 * time.Hour + +// Data is a manifest's `data` section. +type Data struct { + // Own is the data this module keeps itself. + Own []DataItem `json:"own,omitempty"` + // Consumers is what it keeps for its consumers, per provision it grants. + Consumers map[string]ConsumerData `json:"consumers,omitempty"` + // KeptBy is what of its own lives with the provider of a provision it requires — its rows, its + // objects — and how precious that is. The provider's protections follow the stricter of its own + // class for its consumers and this. + KeptBy map[string]KeptData `json:"kept-by,omitempty"` +} + +// DataItem is one piece of a module's own data. +type DataItem struct { + // ID names it within the module: what `data`, `cleanup` and a condition call it. + ID string `json:"id"` + // Path is one of the module's directories, `${dir:}`, or an operator's path it was given, + // `${access:}`, or a path beneath either. Never a machine path (novox/hq ADR 0112). + Path string `json:"path"` + Class string `json:"class"` + // Backup is how it is copied: "copy" (the holder reads it as it stands), "none", or a dump — a + // command that writes a consistent copy into another item, which is copied. Unsaid, anything but a + // cache is copied, unless it says it is protected by redundancy instead. + Backup *DataBackup `json:"backup,omitempty"` + // Redundancy says it is protected by the redundancy of the storage it lives on rather than by a + // copy, and why that is enough: the media library on an array there is no room to copy. The + // backup holder then watches that array, and a degraded one is said. + Redundancy string `json:"redundancy,omitempty"` + // Within is how old its last good backup may be; unsaid, DefaultBackupWithin. + Within string `json:"within,omitempty"` + // Measure is how the backup holder measures it: `walk` (the default — every file, at most daily and + // bounded, for small items), `dataset` (a ZFS dataset's own counters, hourly, nothing walked) or + // `shallow` (its top-level entries only, no size). A large item never says walk. + Measure string `json:"measure,omitempty"` + // Active is how long it may go unwritten before that is a fault — for data something is + // expected to write all the time. Unsaid, a quiet item is not a fault. + Active string `json:"active,omitempty"` + // Why is a line for the reviewer: why this class. + Why string `json:"why,omitempty"` +} + +// ConsumerData is what a provider keeps for the consumers of one provision. +type ConsumerData struct { + Class string `json:"class"` + // In is where it lives: one of the module's own items (whose protection covers it), or a + // provision this module requires (whose provider keeps it, as its consumer). Unsaid only for none + // and cache. + In string `json:"in,omitempty"` + Why string `json:"why,omitempty"` +} + +// KeptData is how precious what a module keeps with a provider is. +type KeptData struct { + Class string `json:"class"` + Why string `json:"why,omitempty"` +} + +// DataBackup is how an item is copied. +type DataBackup struct { + Copy bool + None bool + // Dump is the command writing a consistent copy into the item Into. + Dump string + Into string +} + +// UnmarshalJSON reads "copy", "none" or {"dump": "...", "into": ""}. +func (b *DataBackup) UnmarshalJSON(raw []byte) error { + var word string + if err := json.Unmarshal(raw, &word); err == nil { + switch word { + case "copy": + *b = DataBackup{Copy: true} + case "none": + *b = DataBackup{None: true} + default: + return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}, not %q", word) + } + return nil + } + var full struct { + Dump string `json:"dump"` + Into string `json:"into"` + } + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&full); err != nil { + return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}: %w", err) + } + *b = DataBackup{Dump: full.Dump, Into: full.Into} + return nil +} + +// MarshalJSON writes it back in the form it was written. +func (b DataBackup) MarshalJSON() ([]byte, error) { + switch { + case b.Copy: + return json.Marshal("copy") + case b.None: + return json.Marshal("none") + } + return json.Marshal(struct { + Dump string `json:"dump"` + Into string `json:"into"` + }{b.Dump, b.Into}) +} + +// IsDump is whether the item is copied by a dump. +func (b *DataBackup) IsDump() bool { return b != nil && b.Dump != "" } + +// BackedUp is whether the holder keeps restore points of this item: anything but a cache by default — +// the nightly backup is the standard plan — unless it says "none", or says it is protected by +// redundancy and says nothing of a backup. +func (it DataItem) BackedUp() bool { + switch { + case it.Backup == nil: + return it.Class != ClassCache && it.Redundancy == "" + case it.Backup.None: + return false + } + return true +} + +// Protection is how the item is protected, in one word: "backup", "redundancy", both joined by "+", +// or "none". +func (it DataItem) Protection() string { + var by []string + if it.BackedUp() { + by = append(by, "backup") + } + if it.Redundancy != "" { + by = append(by, "redundancy") + } + if len(by) == 0 { + return "none" + } + return strings.Join(by, "+") +} + +// The ways an item is measured. +const ( + MeasureWalk = "walk" + MeasureDataset = "dataset" + MeasureShallow = "shallow" +) + +// MeasuredBy is how the item is measured, with the default applied. +func (it DataItem) MeasuredBy() string { + if it.Measure == "" { + return MeasureWalk + } + return it.Measure +} + +// OwnedByModule is whether the item is in one of the module's own directories — the mesh's to retire — +// rather than an operator's path it was given, which the mesh never retires and never deletes. +func (it DataItem) OwnedByModule() bool { return strings.HasPrefix(it.Path, "${dir:") } + +// BackupWithin is the item's bound on its last good backup. +func (it DataItem) BackupWithin() time.Duration { + if d, err := ParseDataDuration(it.Within); err == nil && d > 0 { + return d + } + return DefaultBackupWithin +} + +// ActiveWithin is how long the item may go unwritten; zero when quiet is not a fault. +func (it DataItem) ActiveWithin() time.Duration { + d, _ := ParseDataDuration(it.Active) + return d +} + +// ParseDataDuration reads "48h", "90m" or "7d"; empty is zero. +func ParseDataDuration(s string) (time.Duration, error) { + s = strings.TrimSpace(s) + if s == "" { + return 0, nil + } + if days, ok := strings.CutSuffix(s, "d"); ok { + n, err := strconv.Atoi(days) + if err != nil || n <= 0 { + return 0, fmt.Errorf("%q is not a number of days", s) + } + return time.Duration(n) * 24 * time.Hour, nil + } + d, err := time.ParseDuration(s) + if err != nil || d <= 0 { + return 0, fmt.Errorf("%q is not a duration (48h, 90m, 7d)", s) + } + return d, nil +} + +// DataItems is the module's own data, in the order declared. +func (m Manifest) DataItems() []DataItem { + if m.Data == nil { + return nil + } + return m.Data.Own +} + +// DataItem is one own item by id. +func (m Manifest) DataItem(id string) (DataItem, bool) { + for _, it := range m.DataItems() { + if it.ID == id { + return it, true + } + } + return DataItem{}, false +} + +// ConsumerDataOf is what this module says it keeps for a provision's consumers, and whether it says. +func (m Manifest) ConsumerDataOf(provision string) (ConsumerData, bool) { + if m.Data == nil { + return ConsumerData{}, false + } + c, ok := m.Data.Consumers[provision] + return c, ok +} + +// KeptByOf is how precious what this module keeps with a provision's provider is, and whether it says. +func (m Manifest) KeptByOf(provision string) (KeptData, bool) { + if m.Data == nil { + return KeptData{}, false + } + k, ok := m.Data.KeptBy[provision] + return k, ok +} + +// keepsByClass is whether a class keeps something a binding must not move away from. +func keepsByClass(class string) bool { + return class == ClassIrreplaceable || class == ClassValuable || class == ClassRebuildable +} + +// dataID is what an item's id may be: it is a token in a condition's key and a word on a line. +var dataID = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + +// dataPathRef is an item's path: one of the module's directories or accesses, and optionally a path +// beneath it. +var dataPathRef = regexp.MustCompile(`^\$\{(dir|access):([a-z0-9][a-z0-9-]*)\}(/[^\s$]*)?$`) + +// dataProblems is what is wrong with the `data` section itself, from the manifest alone: judged at +// registration as every other per-manifest problem is. Whether a module declares what it should is +// the catalogue check's (DataProblems), because a module already running was written before it. +func (m Manifest) dataProblems() []string { + if m.Data == nil { + return nil + } + var problems []string + say := func(format string, args ...any) { + problems = append(problems, fmt.Sprintf("%s's data: ", m.Module)+fmt.Sprintf(format, args...)) + } + dirs := map[string]bool{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) == "directory" { + dirs[fmt.Sprint(r["id"])] = true + } + } + accesses := map[string]bool{} + for _, a := range m.Accesses { + if a.ID != "" { + accesses[a.ID] = true + } + } + ids := map[string]DataItem{} + paths := map[string]string{} + for i, it := range m.Data.Own { + label := it.ID + if label == "" { + label = fmt.Sprintf("item %d", i+1) + } + if !dataID.MatchString(it.ID) { + say("%s has no usable id: lower-case letters, digits and dashes", label) + } else if _, twice := ids[it.ID]; twice { + say("%s is declared twice", it.ID) + } + ids[it.ID] = it + ref := dataPathRef.FindStringSubmatch(it.Path) + switch { + case ref == nil: + say("%s's path %q is not one of the module's directories or accesses: ${dir:} or ${access:}, "+ + "or a path beneath one (a definition names no machine path, novox/hq ADR 0112)", label, it.Path) + case ref[1] == "dir" && !dirs[ref[2]]: + say("%s's path names ${dir:%s}, and %s declares no directory %q", label, ref[2], m.Module, ref[2]) + case ref[1] == "access" && !accesses[ref[2]]: + say("%s's path names ${access:%s}, and %s declares no access %q", label, ref[2], m.Module, ref[2]) + case strings.Contains(it.Path, ".."): + say("%s's path %q climbs out of its directory", label, it.Path) + } + if other, twice := paths[it.Path]; twice && it.Path != "" { + say("%s and %s name the same path %s", other, label, it.Path) + } + paths[it.Path] = label + switch it.Class { + case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache: + case ClassNone: + say("%s is class none, which only a provision keeping nothing of its consumers' is; own data "+ + "that is disposable is cache", label) + default: + say("%s's class %q is not one the mesh protects by: irreplaceable, valuable, rebuildable or cache", + label, it.Class) + } + if it.Class == ClassIrreplaceable && !it.BackedUp() && strings.TrimSpace(it.Redundancy) == "" { + say("%s is irreplaceable and is neither backed up nor said to be protected by redundancy; the only "+ + "copy of something is protected one way or the other (novox/hq ADR 0233) — copy it, dump it into "+ + "another item, or say `redundancy` with why that is enough", label) + } + if it.Class == ClassCache && it.Backup != nil && !it.Backup.None { + say("%s is a cache and asks to be backed up; a cache is disposable, or it is not a cache", label) + } + if it.Class == ClassCache && it.Redundancy != "" { + say("%s is a cache and says it is protected by redundancy; a cache is not protected", label) + } + switch it.Measure { + case "", MeasureWalk, MeasureDataset, MeasureShallow: + default: + say("%s's measure %q is walk, dataset or shallow", label, it.Measure) + } + if _, err := ParseDataDuration(it.Within); err != nil { + say("%s's within: %v", label, err) + } + if _, err := ParseDataDuration(it.Active); err != nil { + say("%s's active: %v", label, err) + } + if it.Within != "" && !it.BackedUp() { + say("%s states within, and is not backed up", label) + } + if it.Active != "" && !Watched(it.Class) { + say("%s is %s and expects writes; only irreplaceable and valuable data is watched", label, it.Class) + } + } + // Dumps go into an item of the same module, declared, and not into themselves. + for _, it := range m.Data.Own { + if it.Backup == nil || it.Backup.Copy || it.Backup.None { + continue + } + switch into, ok := ids[it.Backup.Into]; { + case strings.TrimSpace(it.Backup.Dump) == "": + say("%s's backup names no dump command", it.ID) + case it.Backup.Into == "": + say("%s's dump says no item it writes into", it.ID) + case !ok: + say("%s's dump writes into %q, which is not one of the module's data items", it.ID, it.Backup.Into) + case into.ID == it.ID: + say("%s's dump writes into itself; a dump is copied from where it lands", it.ID) + case into.Class == ClassCache: + say("%s's dump writes into %s, a cache; what is copied must not be disposable", it.ID, into.ID) + } + if it.Backup.Dump != "" { + declared := map[string]string{} + for d := range dirs { + declared[d] = "" + } + if _, err := dirFill(it.Backup.Dump, declared, m.Module); err != nil { + say("%s's dump: %v", it.ID, err) + } + } + } + provided := map[string]bool{} + for _, o := range m.Provides { + provided[o.Name] = true + } + wants := map[string]bool{} + for _, w := range m.Wants() { + wants[w] = true + } + for _, provision := range sortedKeys(m.Data.Consumers) { + c := m.Data.Consumers[provision] + if !provided[provision] { + say("says what it keeps for the consumers of %q, which it does not provide", provision) + } + switch c.Class { + case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache, ClassNone: + default: + say("its consumers' %s is class %q; one of irreplaceable, valuable, rebuildable, cache or none", provision, c.Class) + } + switch { + case c.Class == ClassNone && c.In != "": + say("its consumers' %s keeps nothing and says where it is kept (%s)", provision, c.In) + case keepsByClass(c.Class) && c.In == "": + say("its consumers' %s is %s and says nowhere it lives: `in` names one of the module's items, or a "+ + "provision it requires", provision, c.Class) + case c.In != "": + if own, ok := ids[c.In]; ok { + if c.Class == ClassIrreplaceable && !own.BackedUp() && own.Redundancy == "" { + say("its consumers' %s is irreplaceable and lives in %s, which is not protected", provision, c.In) + } + if classRank[own.Class] < classRank[c.Class] { + say("its consumers' %s is %s and lives in %s, which is only %s", provision, c.Class, c.In, own.Class) + } + } else if !wants[c.In] { + say("its consumers' %s lives in %q, which is neither one of its data items nor a provision it "+ + "requires", provision, c.In) + } + } + } + for _, provision := range sortedKeys(m.Data.KeptBy) { + k := m.Data.KeptBy[provision] + if !wants[provision] { + say("says it keeps data with the provider of %q, which it does not require", provision) + } + switch k.Class { + case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache: + default: + say("what it keeps with %s is class %q; one of irreplaceable, valuable, rebuildable or cache", provision, k.Class) + } + } + return problems +} + +// KeepsConsumerData is whether this module, providing a provision, keeps what each consumer writes +// there (novox/hq ADR 0232, ADR 0233): whether a consumer bound to it is bound to its data. +// +// **What the data section says, it gets**: irreplaceable, valuable or rebuildable keeps; cache and none +// do not — a cache lost in a move costs speed, not data. Before the section, an offer said it with +// `keeps-consumer-data`, which is still read; unsaid in both, a provider that grants each consumer a +// credential of its own keeps that consumer's data (ADR 0232 §1). The catalogue check refuses the +// unsaid case for a provider that grants (DataProblems), so the inference is what an older definition +// on the shelf gets, never a new one. +func (m Manifest) KeepsConsumerData(provision string) bool { + if c, ok := m.ConsumerDataOf(provision); ok { + return keepsByClass(c.Class) + } + for _, o := range m.Provides { + if o.Name == provision && o.KeepsConsumerData != nil { + return *o.KeepsConsumerData + } + } + _, grants := m.Grants[provision] + return grants +} + +// NeedsBackupHolder is whether a module's data needs the machine's backup holder to be there: it keeps +// something irreplaceable — backed up by the holder, or protected by an array the holder watches. A +// module keeping only valuable or rebuildable data is backed up where a holder is, and refused nowhere +// for want of one: the standard plan, not a requirement. +func (m Manifest) NeedsBackupHolder() bool { + for _, it := range m.DataItems() { + if it.Class == ClassIrreplaceable { + return true + } + } + return false +} + +// --- derived: the backup holder's lines --------------------------------------------------------- + +// The node-backup seat's kinds (novox/hq to-be 43, ADR 0233): `backup` is what to run and which paths +// to keep, `data` every item with its class and protection, for the holder to measure and watch. +const ( + BackupKindBackup = "backup" + BackupKindData = "data" +) + +// derivedContributions is what a module's data section gives the backup holder: its backup lines and +// its items. Every item is listed, so a valuable one on a machine is measured whether or not it is +// copied; a cache is listed and not measured. +func (m Manifest) derivedContributions() []SeatContribution { + items := m.DataItems() + if len(items) == 0 { + return nil + } + var lines []string + kept := map[string]bool{} + keep := func(path string) { + if !kept[path] { + kept[path] = true + lines = append(lines, "path "+path) + } + } + for _, it := range items { + if !it.BackedUp() { + continue + } + if it.Backup.IsDump() { + lines = append(lines, "run "+strings.TrimSpace(it.Backup.Dump)) + if into, ok := m.DataItem(it.Backup.Into); ok { + keep(into.Path) + } + continue + } + keep(it.Path) + } + var described []string + for _, it := range items { + covered := "-" + switch { + case it.Backup.IsDump(): + if into, ok := m.DataItem(it.Backup.Into); ok { + covered = into.Path + } + case it.BackedUp(): + covered = it.Path + } + described = append(described, fmt.Sprintf("item %s %s %s %s %s %s", it.ID, it.Class, it.Path, covered, + it.Protection(), it.MeasuredBy())) + } + var out []SeatContribution + if len(lines) > 0 { + out = append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindBackup, Content: strings.Join(lines, "\n") + "\n"}) + } + return append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindData, Content: strings.Join(described, "\n") + "\n"}) +} + +// allContributions is every contribution a module makes to a seat's holder: what it wrote, and what +// its data section derives. **One list**: a module that declares its data has its backup lines +// composed from it, and a backup line it still writes by hand is not placed — the catalogue check +// refuses it — so the holder never copies two lists that disagree. +func (m Manifest) allContributions() []SeatContribution { + if m.Data == nil { + return m.Contributions + } + derived := m.derivedContributions() + out := make([]SeatContribution, 0, len(m.Contributions)+len(derived)) + for _, c := range m.Contributions { + if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat { + continue + } + out = append(out, c) + } + return append(out, derived...) +} + +// --- the catalogue check ------------------------------------------------------------------------ + +// DataProblems is what the catalogue check refuses about the data a module declares (novox/hq ADR +// 0233), judged over the manifests given together: +// +// - a provider that grants a provision says what it keeps for that provision's consumers; +// - nobody says it on the offer any more (`keeps-consumer-data`): the data section is the one place; +// - nobody writes a backup line by hand: it is derived from the data section; +// - a directory a container writes, mounted whole, is a data item of some class; +// - consumer data said to live in a required provision lives where that provision's provider keeps +// its consumers' data, as preciously, when the provider is given; +// - data a consumer keeps with a provider as irreplaceable is protected there, when the provider is +// given. +// +// **The check's, not registration's**, as ADR 0214's backup rule was: a module already on the shelf +// was written before the rule, and refusing it there would refuse the very providers whose data the +// rule protects. Each module meets it where it is written. +func DataProblems(shelf Shelf) []string { + var problems []string + for _, name := range shelfOrder(shelf) { + m := shelf[name] + for _, provision := range sortedKeys(m.Grants) { + if _, said := m.ConsumerDataOf(provision); !said { + problems = append(problems, fmt.Sprintf( + "%s grants %s and does not say what it keeps for its consumers: data.consumers.%s with a "+ + "class — irreplaceable, valuable, rebuildable, cache, or none (novox/hq ADR 0233)", name, provision, provision)) + } + } + for _, o := range m.Provides { + if o.KeepsConsumerData != nil { + problems = append(problems, fmt.Sprintf( + "%s says keeps-consumer-data on its offer of %s; it is said once, in data.consumers.%s, with a "+ + "class (novox/hq ADR 0233)", name, o.Name, o.Name)) + } + } + for i, c := range m.Contributions { + if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat { + problems = append(problems, fmt.Sprintf( + "%s's contribution %d is a backup line written by hand; backups are derived from the data "+ + "section — declare the data, with its class and how it is protected (novox/hq ADR 0233)", name, i+1)) + } + } + for _, dir := range writtenDirectories(m) { + if !coversDirectory(m, dir) { + problems = append(problems, fmt.Sprintf( + "%s mounts its directory %q into a container to be written, and declares no data in it: "+ + "data.own with a class — cache if it is disposable (novox/hq ADR 0233)", name, dir)) + } + } + if m.Data == nil { + continue + } + for _, provision := range sortedKeys(m.Data.Consumers) { + c := m.Data.Consumers[provision] + if c.In == "" { + continue + } + if _, own := m.DataItem(c.In); own { + continue + } + for _, pname := range shelfOrder(shelf) { + p := shelf[pname] + pc, said := p.ConsumerDataOf(c.In) + if !said || !providesName(p, c.In) { + continue + } + if classRank[pc.Class] < classRank[c.Class] { + problems = append(problems, fmt.Sprintf( + "%s keeps its consumers' %s, %s, in %s — and %s keeps its consumers' %s as %s, "+ + "so it is not protected as %s", name, provision, c.Class, c.In, pname, c.In, pc.Class, c.Class)) + } + } + } + for _, provision := range sortedKeys(m.Data.KeptBy) { + k := m.Data.KeptBy[provision] + if k.Class != ClassIrreplaceable { + continue + } + for _, pname := range shelfOrder(shelf) { + p := shelf[pname] + pc, said := p.ConsumerDataOf(provision) + if !said || !providesName(p, provision) { + continue + } + if !keepsByClass(pc.Class) { + problems = append(problems, fmt.Sprintf( + "%s keeps irreplaceable data with %s, and %s keeps its consumers' %s as %s — nothing of it is "+ + "protected there", name, provision, pname, provision, pc.Class)) + continue + } + if in, own := p.DataItem(pc.In); own && !in.BackedUp() && in.Redundancy == "" { + problems = append(problems, fmt.Sprintf( + "%s keeps irreplaceable data with %s, and %s keeps it in %s, which is neither backed up nor "+ + "on declared redundancy", name, provision, pname, pc.In)) + } + } + } + } + return problems +} + +func providesName(m Manifest, provision string) bool { + for _, o := range m.Provides { + if o.Name == provision { + return true + } + } + return false +} + +// writtenDirectories are the module's directories a container mounts whole and may write: a volume +// `${dir:}:` without `:ro`, or a directory stated by an absolute path mounted the same +// way. A file beneath a directory, or a read-only mount, is configuration the mesh wrote. +func writtenDirectories(m Manifest) []string { + absolute := map[string]string{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "directory" { + continue + } + if p, ok := r["path"].(string); ok && strings.HasPrefix(p, "/") { + absolute[strings.TrimRight(p, "/")] = fmt.Sprint(r["id"]) + } + } + seen := map[string]bool{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "container" { + continue + } + vols, _ := r["volumes"].([]any) + for _, v := range vols { + s, _ := v.(string) + mount := volumeMount.FindStringSubmatch(s) + if mount == nil || volumeReadOnly(mount[3]) { + continue + } + src := strings.TrimRight(mount[1], "/") + if ref := dirPlain.FindStringSubmatch(src); ref != nil { + seen[ref[1]] = true + } else if id, ok := absolute[src]; ok { + seen[id] = true + } + } + } + out := make([]string, 0, len(seen)) + for id := range seen { + out = append(out, id) + } + sort.Strings(out) + return out +} + +// volumeMount is a container's volume, `:[:]`, its source possibly a +// `${dir:}` — whose own colon is not the separator. +var volumeMount = regexp.MustCompile(`^(\$\{(?:dir|access):[a-z0-9][a-z0-9-]*\}[^:]*|[^:]+):([^:]+)(?::(.*))?$`) + +// volumeReadOnly is whether a volume's options mount it read-only. +func volumeReadOnly(options string) bool { + for _, o := range strings.Split(options, ",") { + if strings.TrimSpace(o) == "ro" { + return true + } + } + return false +} + +// dirPlain is a whole directory, `${dir:}` and nothing after it. +var dirPlain = regexp.MustCompile(`^\$\{dir:([a-z0-9][a-z0-9-]*)\}$`) + +// coversDirectory is whether a data item names the directory, a path within it, or a directory it +// is placed beneath. +func coversDirectory(m Manifest, dir string) bool { + parents := map[string]string{} + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "directory" { + continue + } + if p, ok := r["path"].(string); ok { + if ref := dirRef.FindStringSubmatch(p); ref != nil && strings.HasPrefix(p, "${dir:") { + parents[fmt.Sprint(r["id"])] = ref[1] + } + } + } + for _, it := range m.DataItems() { + ref := dataPathRef.FindStringSubmatch(it.Path) + if ref == nil || ref[1] != "dir" { + continue + } + for d, hops := dir, 0; d != "" && hops < 4; d, hops = parents[d], hops+1 { + if ref[2] == d { + return true + } + } + } + return false +} diff --git a/internal/catalogue/data_test.go b/internal/catalogue/data_test.go new file mode 100644 index 0000000..6e55f21 --- /dev/null +++ b/internal/catalogue/data_test.go @@ -0,0 +1,227 @@ +package catalogue + +import ( + "slices" + "strings" + "testing" +) + +// A store as a module now declares it (novox/hq ADR 0233): its own data by directory with a class and a +// dump, and what it keeps for its consumers. +const declaredStore = `{"module":"pg","version":"1", + "provides":[{"name":"postgres-database","scope":"mesh"}], + "grants":{"postgres-database":"${dir:grants}"}, + "data":{ + "own":[ + {"id":"store","path":"${dir:store}","class":"irreplaceable","backup":{"dump":"dump-all > ${dir:dumps}/all","into":"dumps"},"active":"1d"}, + {"id":"dumps","path":"${dir:dumps}","class":"rebuildable","backup":"none"}], + "consumers":{"postgres-database":{"class":"irreplaceable","in":"store"}}}, + "resources":[ + {"id":"grants","type":"directory","mode":"0700"}, + {"id":"store","type":"directory","path":"/srv/store","mode":"0700"}, + {"id":"dumps","type":"directory","path":"${dir:store}/dumps","mode":"0700"}, + {"id":"server","type":"container","image":"x@sha256:aa","volumes":["${dir:store}:/var/lib/postgresql/data"]}]}` + +func mustParse(t *testing.T, raw string) Manifest { + t.Helper() + m, err := ParseManifest([]byte(raw)) + if err != nil { + t.Fatalf("refused: %v", err) + } + return m +} + +func TestADeclaredStoreParsesAndPassesTheCheck(t *testing.T) { + m := mustParse(t, declaredStore) + if problems := DataProblems(Shelf{"pg": m}); len(problems) > 0 { + t.Fatalf("a store declaring its data was refused: %v", problems) + } + if !m.KeepsConsumerData("postgres-database") { + t.Fatal("an irreplaceable consumer class does not keep the consumer's data") + } + if it, _ := m.DataItem("store"); !it.BackedUp() || it.BackupWithin() != DefaultBackupWithin || it.ActiveWithin().Hours() != 24 { + t.Fatalf("the store item reads %+v", it) + } + if it, _ := m.DataItem("dumps"); it.BackedUp() { + t.Fatal("a rebuildable item that says none is backed up") + } +} + +// Every rule of the section itself, refused at parse in the words of the module. +func TestTheDataSectionIsRefusedWhereItIsWrong(t *testing.T) { + for _, c := range []struct{ name, data, want string }{ + {"a class it does not know", `{"own":[{"id":"a","path":"${dir:d}","class":"precious"}]}`, "not one the mesh protects by"}, + {"an access it does not have", `{"own":[{"id":"a","path":"${access:media}","class":"irreplaceable","redundancy":"an array"}]}`, "declares no access"}, + {"irreplaceable and protected by nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":"none"}]}`, "protected one way or the other"}, + {"a cache on redundancy", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","redundancy":"an array"}]}`, "a cache is not protected"}, + {"kept with something not required", `{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}`, "does not require"}, + {"a measure it does not know", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","measure":"du"}]}`, "walk, dataset or shallow"}, + {"a cache expecting writes", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","active":"1d"}]}`, "only irreplaceable and valuable"}, + {"a machine path", `{"own":[{"id":"a","path":"/srv/a","class":"cache"}]}`, "names no machine path"}, + {"a directory it does not declare", `{"own":[{"id":"a","path":"${dir:nowhere}","class":"cache"}]}`, "declares no directory"}, + {"a cache backed up", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","backup":"copy"}]}`, "disposable"}, + {"a dump into nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":{"dump":"x","into":"b"}}]}`, "not one of the module's data items"}, + {"a dump into itself", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":{"dump":"x","into":"a"}}]}`, "into itself"}, + {"a bad duration", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","within":"soon"}]}`, "not a duration"}, + {"the same id twice", `{"own":[{"id":"a","path":"${dir:d}","class":"cache"},{"id":"a","path":"${dir:d}/x","class":"cache"}]}`, "declared twice"}, + {"own data of class none", `{"own":[{"id":"a","path":"${dir:d}","class":"none"}]}`, "only a provision"}, + {"consumers of something not provided", `{"consumers":{"s3-bucket":{"class":"cache"}}}`, "does not provide"}, + {"consumer data that lives nowhere", `{"consumers":{"q":{"class":"irreplaceable"}}}`, "says nowhere it lives"}, + {"consumer data in an item less precious", `{"own":[{"id":"a","path":"${dir:d}","class":"rebuildable"}],"consumers":{"q":{"class":"irreplaceable","in":"a"}}}`, "which is only rebuildable"}, + {"consumer data in something unknown", `{"consumers":{"q":{"class":"rebuildable","in":"elsewhere"}}}`, "neither one of its data items"}, + } { + raw := `{"module":"m","version":"1","provides":["q"],"resources":[{"id":"d","type":"directory","mode":"0700"}],"data":` + c.data + `}` + _, err := ParseManifest([]byte(raw)) + if err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: %v, want %q", c.name, err, c.want) + } + } +} + +// The catalogue check: a granting provider says what it keeps, says it in one place, writes no backup +// line by hand, and every directory a container writes is declared (novox/hq ADR 0233). +func TestTheCatalogueCheckAsksEveryModuleForItsData(t *testing.T) { + unsaid := mustParse(t, `{"module":"q","version":"1","provides":[{"name":"queue","scope":"mesh"}], + "grants":{"queue":"${dir:g}"},"resources":[{"id":"g","type":"directory","mode":"0700"}]}`) + onTheOffer := mustParse(t, `{"module":"r","version":"1","provides":[{"name":"resolver","scope":"mesh","keeps-consumer-data":false}]}`) + byHand := mustParse(t, `{"module":"h","version":"1","resources":[{"id":"d","type":"directory","mode":"0700"}], + "contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:d}"}]}`) + writes := mustParse(t, `{"module":"w","version":"1","resources":[{"id":"d","type":"directory","mode":"0700"}, + {"id":"c","type":"directory","mode":"0700"}, + {"id":"app","type":"container","image":"x@sha256:aa","volumes":["${dir:d}:/data","${dir:c}:/etc/app:ro","${dir:d}/f:/f"]}]}`) + problems := strings.Join(DataProblems(Shelf{"q": unsaid, "r": onTheOffer, "h": byHand, "w": writes}), "\n") + for _, want := range []string{ + "q grants queue and does not say what it keeps", + "r says keeps-consumer-data on its offer", + "h's contribution 1 is a backup line written by hand", + `w mounts its directory "d" into a container to be written`, + } { + if !strings.Contains(problems, want) { + t.Errorf("the check does not say %q:\n%s", want, problems) + } + } + if strings.Contains(problems, `"c"`) { + t.Errorf("a read-only mount was taken for written data:\n%s", problems) + } + // The same module declaring the directory, as a cache, passes. + declared := mustParse(t, `{"module":"w","version":"1","data":{"own":[{"id":"d","path":"${dir:d}","class":"cache"}]}, + "resources":[{"id":"d","type":"directory","mode":"0700"}, + {"id":"app","type":"container","image":"x@sha256:aa","volumes":["${dir:d}:/data"]}]}`) + if p := DataProblems(Shelf{"w": declared}); len(p) > 0 { + t.Fatalf("a declared directory is still refused: %v", p) + } +} + +// Consumer data said to live in a provision the module requires is protected only as well as that +// provision's provider protects its consumers' data. +func TestConsumerDataInARequiredProvisionIsJudgedAgainstItsProvider(t *testing.T) { + idp := mustParse(t, `{"module":"idp","version":"1","requires":["postgres-database"], + "provides":[{"name":"oidc-client","scope":"mesh"}],"grants":{"oidc-client":"${dir:g}"}, + "resources":[{"id":"g","type":"directory","mode":"0700"}], + "data":{"consumers":{"oidc-client":{"class":"irreplaceable","in":"postgres-database"}}}}`) + cachy := mustParse(t, strings.Replace(declaredStore, `"consumers":{"postgres-database":{"class":"irreplaceable","in":"store"}}`, + `"consumers":{"postgres-database":{"class":"cache"}}`, 1)) + if p := strings.Join(DataProblems(Shelf{"idp": idp, "pg": cachy}), "\n"); !strings.Contains(p, "not protected as irreplaceable") { + t.Fatalf("irreplaceable data kept in a cache passed: %s", p) + } + if p := DataProblems(Shelf{"idp": idp, "pg": mustParse(t, declaredStore)}); len(p) > 0 { + t.Fatalf("refused against a provider that keeps its consumers' data: %v", p) + } +} + +// What a binding is sticky by follows the class (ADR 0232, generalised by 0233): a cache's consumers +// move freely, a store's do not; an older definition saying nothing still follows its grant. +func TestKeepingConsumerDataFollowsTheClass(t *testing.T) { + for class, keeps := range map[string]bool{"irreplaceable": true, "valuable": true, "rebuildable": true, "cache": false, "none": false} { + in := `,"in":"d"` + own := `"own":[{"id":"d","path":"${dir:d}","class":"irreplaceable"}],` + if !keeps { + in, own = "", "" + } + m := mustParse(t, `{"module":"p","version":"1","provides":[{"name":"q","scope":"mesh"}],"grants":{"q":"${dir:d}"}, + "resources":[{"id":"d","type":"directory","mode":"0700"}], + "data":{`+own+`"consumers":{"q":{"class":"`+class+`"`+in+`}}}}`) + if m.KeepsConsumerData("q") != keeps { + t.Errorf("class %s keeps consumer data: %v, want %v", class, !keeps, keeps) + } + shelf := map[string]Manifest{"p": m} + if KeepsConsumerData(shelf, "q") != keeps { + t.Errorf("class %s: the provision by name keeps: %v", class, !keeps) + } + } + older := mustParse(t, `{"module":"p","version":"1","provides":[{"name":"q","scope":"mesh"}],"grants":{"q":"${dir:d}"}, + "resources":[{"id":"d","type":"directory","mode":"0700"}]}`) + if !older.KeepsConsumerData("q") { + t.Fatal("an older definition that grants no longer keeps its consumers' data") + } +} + +// The backup holder's lines are derived: the dump runs and the directory it writes into is kept, +// anything but a cache is copied by default — the standard nightly plan — an item on redundancy is not +// copied, and every item is listed with its class and protection for the holder to measure and watch: +// directories filled, a home directory filled from the machine, an operator's path from where the +// assignment placed it. A backup line still written by hand beside a data section is not placed. +func TestTheBackupHoldersLinesAreDerivedFromTheData(t *testing.T) { + pg := mustParse(t, declaredStore) + pg.Contributions = append(pg.Contributions, SeatContribution{Seat: BackupSeat, Kind: "backup", Content: "path /stale"}) + agent := mustParse(t, `{"module":"agent","version":"1", + "data":{"own":[{"id":"home","path":"${dir:home}","class":"valuable"},{"id":"tmp","path":"${dir:home}/tmp","class":"cache"}]}, + "resources":[{"id":"home","type":"directory","path":"${machine:account-home}/.agent","mode":"0700"}]}`) + media := mustParse(t, `{"module":"media","version":"1", + "accesses":[{"id":"films","mode":"read"}], + "data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"on a redundant array; no room for a copy","measure":"dataset"}, + {"id":"meta","path":"${dir:meta}","class":"rebuildable"}]}, + "resources":[{"id":"meta","type":"directory","mode":"0700"}]}`) + facts := map[string]string{"account-home": "/home/op"} + with := Rendering{Settings: SettingsBy{"media": {{From: "node", Values: map[string]any{AccessesSetting: map[string]any{"films": "/tank/films"}}}}}} + backup, err := seatContributions([]Manifest{pg, agent, media}, BackupSeat, BackupKindBackup, with, facts) + if err != nil { + t.Fatal(err) + } + want := "# agent\npath /home/op/.agent\n# media\npath /var/lib/media/meta\n# pg\nrun dump-all > /srv/store/dumps/all\npath /srv/store/dumps\n" + if backup != want { + t.Fatalf("backup lines:\n%s\nwant:\n%s", backup, want) + } + data, err := seatContributions([]Manifest{pg, agent, media}, BackupSeat, BackupKindData, with, facts) + if err != nil { + t.Fatal(err) + } + want = "# agent\nitem home valuable /home/op/.agent /home/op/.agent backup walk\nitem tmp cache /home/op/.agent/tmp - none walk\n" + + "# media\nitem films irreplaceable /tank/films - redundancy dataset\nitem meta rebuildable /var/lib/media/meta /var/lib/media/meta backup walk\n" + + "# pg\nitem store irreplaceable /srv/store /srv/store/dumps backup walk\nitem dumps rebuildable /srv/store/dumps - none walk\n" + if data != want { + t.Fatalf("data lines:\n%s\nwant:\n%s", data, want) + } + if _, err := seatContributions([]Manifest{agent}, BackupSeat, BackupKindData, Rendering{}, nil); err == nil { + t.Fatal("a home directory with no account on the machine reached the holder as a placeholder") + } + // What keeps something irreplaceable depends on the machine's backup holder — it backs it up or + // watches its array; valuable data alone is backed up where a holder is, and refused nowhere. + if !slices.Contains(DependsOn(media), BackupSeat) || !slices.Contains(DependsOn(pg), BackupSeat) || + slices.Contains(DependsOn(agent), BackupSeat) { + t.Fatalf("depends: media %v, pg %v, agent %v", DependsOn(media), DependsOn(pg), DependsOn(agent)) + } + if it, _ := media.DataItem("films"); it.OwnedByModule() || it.Protection() != "redundancy" { + t.Fatalf("an operator's path reads %+v", it) + } +} + +// Data a consumer keeps with a provider as irreplaceable must be protected there: the check refuses a +// provider that keeps its consumers' data as a cache, or in an item with no protection. +func TestIrreplaceableDataKeptWithAProviderMustBeProtectedThere(t *testing.T) { + photos := mustParse(t, `{"module":"photos","version":"1","requires":["s3-bucket"], + "data":{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}}`) + store := func(backup string) Manifest { + return mustParse(t, `{"module":"objects","version":"1","provides":[{"name":"s3-bucket","scope":"mesh"}], + "grants":{"s3-bucket":"${dir:g}"}, + "data":{"own":[{"id":"data","path":"${dir:data}","class":"valuable","backup":"`+backup+`"}], + "consumers":{"s3-bucket":{"class":"valuable","in":"data"}}}, + "resources":[{"id":"g","type":"directory","mode":"0700"},{"id":"data","type":"directory","mode":"0700"}]}`) + } + if p := DataProblems(Shelf{"photos": photos, "objects": store("copy")}); len(p) > 0 { + t.Fatalf("a provider backing its consumers' data up was refused: %v", p) + } + if p := strings.Join(DataProblems(Shelf{"photos": photos, "objects": store("none")}), "\n"); !strings.Contains(p, "neither backed up") { + t.Fatalf("irreplaceable photos in an unprotected store passed: %s", p) + } +} diff --git a/internal/catalogue/environment_into.go b/internal/catalogue/environment_into.go index 52f662e..2cb6c17 100644 --- a/internal/catalogue/environment_into.go +++ b/internal/catalogue/environment_into.go @@ -576,7 +576,7 @@ func contributionsInto(resource map[string]any, m Manifest, modules []Manifest, found := ofContributed.FindStringSubmatch(placeholder) first, second, _ := strings.Cut(found[2], ":") if found[1] == "contribution" { - placed, err := seatContributions(modules, first, second, with) + placed, err := seatContributions(modules, first, second, with, facts) if err != nil && failed == nil { failed = err } diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 394dd4d..b2ca58c 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -239,25 +239,6 @@ func (m Manifest) IdentityBoundOf(provision string) IdentityBound { return IdentityBound{} } -// KeepsConsumerData is whether this module, providing a provision, keeps what each consumer writes -// there (novox/hq ADR 0232): whether a consumer bound to it is bound to its data. -// -// **What an offer says, it gets.** Unsaid, it follows from whether the provider grants each consumer -// a credential of its own: a provider that does makes an account for every consumer — a role and its -// database, a key and its bucket, a client — and what the consumer writes under that account stays -// with that provider. One that grants nothing keeps nothing of anybody's: the resolver, a CA, the -// artifact store each answer any consumer alike, and moving a consumer between two of them loses -// nothing. -func (m Manifest) KeepsConsumerData(provision string) bool { - for _, o := range m.Provides { - if o.Name == provision && o.KeepsConsumerData != nil { - return *o.KeepsConsumerData - } - } - _, grants := m.Grants[provision] - return grants -} - // KeepsConsumerData is whether a provision, by name, keeps its consumers' data across the // catalogue: true when any module providing it at the mesh's scope does. **A property of the name**, // as brokering is: two providers disagreeing would make the same binding sticky or free depending on @@ -466,6 +447,12 @@ type Manifest struct { // (novox/hq ADR 0201). Not history — that is an event — and never a secret, sealed or not. State []StateDeclaration `json:"state,omitempty"` + // Data is every kind of data this module keeps — its own, by directory, and what it keeps for + // its consumers, by provision — each with a class the mesh protects and watches it by (novox/hq + // ADR 0233). One list: the backup holder's lines, the bindings that do not move, what an + // unassignment retires and what the self-check measures are all derived from it. + Data *Data `json:"data,omitempty"` + // Reads are other modules' state this module reads and watches, each `.` // (novox/hq ADR 0201). Read-only: only the owner's instances write. Reads []string `json:"reads,omitempty"` @@ -2007,6 +1994,7 @@ func ParseManifest(raw []byte) (Manifest, error) { problems = append(problems, m.shellProblems()...) problems = append(problems, m.contributionPlaceholderProblems()...) problems = append(problems, m.seatContributionProblems()...) + problems = append(problems, m.dataProblems()...) for i, r := range m.Resources { id, _ := r["id"].(string) diff --git a/internal/catalogue/seat_contributions.go b/internal/catalogue/seat_contributions.go index fbd51b1..1236512 100644 --- a/internal/catalogue/seat_contributions.go +++ b/internal/catalogue/seat_contributions.go @@ -23,46 +23,6 @@ const MessageBusSeat = "node-message-bus" // BackupSeat is the machine's backups (novox/hq ADR 0214, to-be 43). const BackupSeat = "node-backup" -// storeProvisions are the provisions whose provider keeps its consumers' data (novox/hq ADR 0214): -// a module providing one must say how to back it up, or the data the mesh hands out is the data it -// cannot restore — issue 241's seven databases. A provision that holds nothing worth keeping (a -// route, a cache, a name) is not here; adding one is adding a store. -var storeProvisions = map[string]bool{ - "postgres-database": true, - "mssql-database": true, - "mongodb-database": true, - "s3-bucket": true, - "influxdb-api": true, - "secret": true, -} - -// CheckBackup is a store provider that contributes no backup (novox/hq ADR 0214, "How it is -// checked"). -// -// **The catalogue check's, not parsing's.** A manifest already registered and running was written -// before the rule; refusing it on read would make the controller refuse the very providers whose -// data the rule protects. New definitions meet it in the catalogue check, where they are written. -func CheckBackup(m Manifest) []string { - backs := false - for _, c := range m.Contributions { - if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat && c.Kind == "backup" { - backs = true - } - } - if backs { - return nil - } - var problems []string - for _, o := range m.Provides { - if storeProvisions[o.Name] { - problems = append(problems, fmt.Sprintf( - "%s provides %s, which keeps its consumers' data, and contributes no backup to %s; a "+ - "store says how its data is copied (novox/hq ADR 0214)", m.Module, o.Name, BackupSeat)) - } - } - return problems -} - // SeatContribution is one piece of configuration a module gives a seat's holder to place. type SeatContribution struct { // Seat is the seat whose holder places it. @@ -188,7 +148,11 @@ func seatPlaceholderProblems(m Manifest, r map[string]any) []string { // comment line naming it in the tool's grammar, and empty when nothing is contributed. A kind that // takes directories has each contributor's `${dir:}` filled with where that module's directory // is on this machine (novox/hq to-be 43). -func seatContributions(modules []Manifest, seat, kind string, with Rendering) (string, error) { +// +// A directory placed in a person's home is named `${machine:account-home}/…` (novox/hq to-be 29); in a +// kind that takes directories that is filled from the machine's facts as well, so the holder reads a +// path, never a placeholder (novox/hq ADR 0233: an item in the operator's home is backed up too). +func seatContributions(modules []Manifest, seat, kind string, with Rendering, facts map[string]string) (string, error) { s, r, ok := receivable(seat, kind) if !ok { return "", nil @@ -197,7 +161,7 @@ func seatContributions(modules []Manifest, seat, kind string, with Rendering) (s var b strings.Builder for _, m := range inModuleOrder(modules) { named := false - for _, c := range m.Contributions { + for _, c := range m.allContributions() { if c.Kind != kind { continue } @@ -214,6 +178,27 @@ func seatContributions(modules []Manifest, seat, kind string, with Rendering) (s if err != nil && failed == nil { failed = err } + // An operator's path the module was given, as an item of data on it (novox/hq ADR 0233). + if accessRef.MatchString(filled) { + _, byID, err := accessesFor(m, with.Settings[m.Module]) + if err == nil { + filled, err = accessFill(filled, byID, m.Module) + } + if err != nil && failed == nil { + failed = err + } + } + for _, key := range machineUsed(filled) { + value, has := facts[key] + if !has { + if failed == nil { + failed = fmt.Errorf("%s's %s for %s says ${machine:%s}, and this machine says %s", + m.Module, kind, s.Name, key, orNothing(namesOfFacts(facts))) + } + continue + } + filled = strings.ReplaceAll(filled, "${machine:"+key+"}", value) + } content = filled } b.WriteString(content) diff --git a/internal/catalogue/seat_dependencies.go b/internal/catalogue/seat_dependencies.go index d9818ce..dc2e0f0 100644 --- a/internal/catalogue/seat_dependencies.go +++ b/internal/catalogue/seat_dependencies.go @@ -123,6 +123,12 @@ func contributedTo(m Manifest) []string { out = append(out, s.Name) } } + // And the backup holder for what is irreplaceable (novox/hq ADR 0233): it backs it up or watches the + // array it is on. What is valuable is backed up where a holder is — the standard plan — and makes + // no machine refuse it for want of one. + if m.NeedsBackupHolder() { + out = append(out, BackupSeat) + } return out } diff --git a/internal/catalogue/seats.go b/internal/catalogue/seats.go index 5bef725..4ebd6b8 100644 --- a/internal/catalogue/seats.go +++ b/internal/catalogue/seats.go @@ -256,8 +256,10 @@ var defaultSeats = append([]Seat{ // disasters. A module contributes `backup` lines — what to run to take a consistent copy, and // which of its directories to keep. {Name: BackupSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0214", - Serves: backupVerbs(), - Receives: []Receivable{{Kind: "backup", Comment: "#", Dirs: true}}}, + Serves: backupVerbs(), + // `backup` is what to run and which paths to keep; `data` every item a module declares, with its + // class, for the holder to measure (novox/hq ADR 0233). Both derived from modules' data sections. + Receives: []Receivable{{Kind: BackupKindBackup, Comment: "#", Dirs: true}, {Kind: BackupKindData, Comment: "#", Dirs: true}}}, // Deferred (novox/hq ADR 0121): renaming to mesh-private-network is a scope + server/client // model change, not a rename, so it stays until that is built. {Name: "the-private-network", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index c54d7c4..33d5f43 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -282,8 +282,10 @@ var ControllerVerbs = []Verb{ "cause": "with answer: the cause in a word (retire-waiting when absent)", }, nil)}, {Name: "cleanup", Description: "Every consumer a provider holds retired — its age, its size where the " + - "backend knows, and why it was retired. With consumer (and node, module): the provider deletes that one " + - "retired consumer — never an active one. With older-than: every retired consumer older than that many " + + "backend knows, and why it was retired — and every module's own data retired on its machine (ADR 0233). " + + "With consumer (and node, module): the provider deletes that one retired consumer — never an active one; " + + "for a module's own retired item, consumer names the item and the machine's backup holder takes a last " + + "restore point of it, then deletes it. With older-than: every retired consumer older than that many " + "days, listed; deleted only with confirm. Deleting is a hand act, which says why (novox/hq ADR 0230).", Input: schema(map[string]string{ "node": "with consumer: the machine the provider runs on", @@ -294,6 +296,15 @@ var ControllerVerbs = []Verb{ "why": "with consumer or older-than: why — required, and recorded in the hand-act log", "cause": "with consumer or older-than: the cause in a word (cleanup-waiting when absent)", }, nil, "confirm")}, + // The data every machine declares (novox/hq ADR 0233). + {Name: "data", Description: "Every item of data every machine declares, as the self-check last measured it: " + + "its class (irreplaceable, rebuildable, cache), where it is, its size, its newest write, its newest good " + + "backup and the bound on it — and what is retired: kept after its module left the machine, removed only by " + + "`cleanup delete` (novox/hq ADR 0233).", + Input: schema(map[string]string{ + "machine": "one machine (optional)", + "retired": "\"true\": only what is retired", + }, nil, "retired")}, {Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " + "`builds` with that id follows it line by line, and the module is registered when the outcome comes.", Input: schema(map[string]string{ diff --git a/internal/inventory/data.go b/internal/inventory/data.go new file mode 100644 index 0000000..3518e57 --- /dev/null +++ b/internal/inventory/data.go @@ -0,0 +1,316 @@ +package inventory + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// The data a module declares, as the mesh found it on each machine (novox/hq ADR 0233). +// +// The self-check composes what every machine declares and asks its backup holder what it measured; +// this keeps both. An irreplaceable item a machine no longer declares — its module unassigned, or no +// longer pulled in — is retired here, not forgotten: kept, with when and why, until a person deletes +// it through `cleanup delete`. + +// DeclaredData is one item a machine's composition declares now. +type DeclaredData struct { + Module, Item, Class string + // Owned is whether it is in the module's own directory: only that is the mesh's to retire. + Owned bool + // Protection is backup, redundancy, both ("backup+redundancy") or none. + Protection string +} + +// Redundancy is the redundant storage an item is on, as its machine's holder read it. +type Redundancy struct { + Kind string `json:"kind"` + Where string `json:"where"` + Healthy *bool `json:"healthy"` + Said string `json:"said"` +} + +// Measurement is what a machine's backup holder said of one item. Nil fields were not measured. +type Measurement struct { + Path string + Size *int64 + LastWrite *time.Time + MeasuredAt *time.Time + LastBackup *time.Time + // Error is what the holder could not measure, when it could not. + Error string + // Precision is what the size is, as the holder said it: "exact", "dataset …", "partial: …", "no size: …". + Precision string + Redundancy *Redundancy +} + +// DataRecord is one item as the mesh keeps it. +type DataRecord struct { + Machine, Module, Item, Class, Path string + Owned bool + Protection string + FirstSeen, DeclaredAt time.Time + MeasureError string + Precision string + Redundancy *Redundancy + Size *int64 + LastWrite, MeasuredAt, LastBackup *time.Time + RetiredAt *time.Time + RetiredWhy string + DeletedAt *time.Time + DeletedBy, DeletedWhy string +} + +// Comparable is whether a size is fit to compare with another: exact, or a dataset's own counters. +func Comparable(precision string) bool { + return precision == "" || precision == "exact" || strings.HasPrefix(precision, "dataset ") +} + +// Dataset is the ZFS dataset a size was read from, when it was: what several items on one dataset share. +func Dataset(precision string) string { + rest, ok := strings.CutPrefix(precision, "dataset ") + if !ok { + return "" + } + name, _, _ := strings.Cut(rest, ":") + return name +} + +// Retired is whether the item is retired and not deleted. +func (r DataRecord) Retired() bool { return r.RetiredAt != nil && r.DeletedAt == nil } + +// Key names it in one string, the way conditions and verbs do: //. +func (r DataRecord) Key() string { return r.Machine + "/" + r.Module + "/" + r.Item } + +// DataChange is what recording a machine's data changed: what it retired and what came back. +type DataChange struct { + Retired, Reenabled []DataRecord +} + +// readingEvery is how often a measurement is kept as a reading: the shrink is read over days, and a +// row every five minutes would say the same thing sixty times an hour. +const readingEvery = 55 * time.Minute + +// readingsKept is how long readings are kept. +const readingsKept = 90 * 24 * time.Hour + +// dataKey is one item's identity on one machine. +type dataKey struct{ module, item string } + +// RecordData keeps what one machine declares now and what its holder measured, at now. +// +// **Only from a composition that worked.** The caller passes the declared set of a machine whose +// composition succeeded; an item missing from it is then really no longer declared. An irreplaceable +// one is retired — never deleted, never forgotten — and anything else is forgotten, because what is +// rebuildable or a cache is not the mesh's to keep track of once its module is gone. An item declared +// again comes back from retirement as it was. +func (i *Inventory) RecordData(ctx context.Context, machine string, declared []DeclaredData, + measured map[string]map[string]Measurement, why string, now time.Time) (DataChange, error) { + var change DataChange + tx, err := i.store.Pool().Begin(ctx) + if err != nil { + return change, err + } + defer tx.Rollback(ctx) //nolint:errcheck + + existing := map[dataKey]DataRecord{} + rows, err := tx.Query(ctx, dataSelect+` where machine = $1`, machine) + if err != nil { + return change, err + } + for rows.Next() { + r, err := scanData(rows) + if err != nil { + rows.Close() + return change, err + } + existing[dataKey{r.Module, r.Item}] = r + } + rows.Close() + if err := rows.Err(); err != nil { + return change, err + } + + seen := map[dataKey]bool{} + for _, d := range declared { + k := dataKey{d.Module, d.Item} + seen[k] = true + m := measured[d.Module][d.Item] + var red struct { + Kind, Where, Said *string + Healthy *bool + } + if r := m.Redundancy; r != nil { + red.Kind, red.Where, red.Said, red.Healthy = &r.Kind, &r.Where, &r.Said, r.Healthy + } + was, had := existing[k] + if had && was.Retired() { + change.Reenabled = append(change.Reenabled, was) + } + // Deleted and declared again is new data: it starts over. + fresh := !had || was.DeletedAt != nil + if _, err := tx.Exec(ctx, ` + insert into data_item (machine, module, item, class, path, first_seen, declared_at, + size_bytes, last_write, measured_at, last_backup, owned, protection, + measure_error, redundancy_kind, redundancy_where, redundancy_healthy, redundancy_said, + precision) + values ($1, $2, $3, $4, $5, $6, $6, $7, $8, $9, $10, $12, $13, $14, $15, $16, $17, $18, $19) + on conflict (machine, module, item) do update set + class = excluded.class, + owned = excluded.owned, + protection = excluded.protection, + precision = case when excluded.measured_at is not null then excluded.precision else data_item.precision end, + size_bytes = case when excluded.measured_at is not null then excluded.size_bytes else data_item.size_bytes end, + measure_error = case when excluded.measured_at is not null then excluded.measure_error else data_item.measure_error end, + redundancy_kind = case when excluded.measured_at is not null then excluded.redundancy_kind else data_item.redundancy_kind end, + redundancy_where = case when excluded.measured_at is not null then excluded.redundancy_where else data_item.redundancy_where end, + redundancy_healthy = case when excluded.measured_at is not null then excluded.redundancy_healthy else data_item.redundancy_healthy end, + redundancy_said = case when excluded.measured_at is not null then excluded.redundancy_said else data_item.redundancy_said end, + path = case when excluded.path <> '' then excluded.path else data_item.path end, + first_seen = case when $11::boolean then excluded.first_seen else data_item.first_seen end, + declared_at = excluded.declared_at, + last_write = coalesce(excluded.last_write, data_item.last_write), + measured_at = coalesce(excluded.measured_at, data_item.measured_at), + last_backup = coalesce(excluded.last_backup, data_item.last_backup), + retired_at = null, retired_why = null, + deleted_at = null, deleted_by = null, deleted_why = null`, + machine, d.Module, d.Item, d.Class, m.Path, now, m.Size, m.LastWrite, m.MeasuredAt, m.LastBackup, + fresh, d.Owned, d.Protection, nullable(m.Error), red.Kind, red.Where, red.Healthy, red.Said, + nullable(m.Precision)); err != nil { + return change, err + } + if m.Size != nil && m.MeasuredAt != nil && Comparable(m.Precision) { + if _, err := tx.Exec(ctx, ` + insert into data_reading (machine, module, item, at, size_bytes, last_write) + select $1, $2, $3, $4, $5, $6 + where not exists (select 1 from data_reading + where machine = $1 and module = $2 and item = $3 and at > $4::timestamptz - $7::interval)`, + machine, d.Module, d.Item, *m.MeasuredAt, *m.Size, m.LastWrite, + fmt.Sprintf("%d seconds", int(readingEvery.Seconds()))); err != nil { + return change, err + } + } + } + for k, r := range existing { + if seen[k] || r.DeletedAt != nil || r.Retired() { + continue + } + if !catalogue.Retires(r.Class) || !r.Owned { + if _, err := tx.Exec(ctx, `delete from data_item where machine = $1 and module = $2 and item = $3`, + machine, k.module, k.item); err != nil { + return change, err + } + continue + } + if _, err := tx.Exec(ctx, `update data_item set retired_at = $4, retired_why = $5 + where machine = $1 and module = $2 and item = $3`, machine, k.module, k.item, now, why); err != nil { + return change, err + } + at := now + r.RetiredAt, r.RetiredWhy = &at, why + change.Retired = append(change.Retired, r) + } + if _, err := tx.Exec(ctx, `delete from data_reading where at < $1`, now.Add(-readingsKept)); err != nil { + return change, err + } + return change, tx.Commit(ctx) +} + +const dataSelect = `select machine, module, item, class, path, first_seen, declared_at, size_bytes, last_write, + measured_at, last_backup, retired_at, coalesce(retired_why, ''), deleted_at, coalesce(deleted_by, ''), + coalesce(deleted_why, ''), owned, protection, coalesce(measure_error, ''), redundancy_kind, redundancy_where, + redundancy_healthy, redundancy_said, coalesce(precision, '') from data_item` + +func scanData(rows pgx.Row) (DataRecord, error) { + var r DataRecord + var kind, where, said *string + var healthy *bool + err := rows.Scan(&r.Machine, &r.Module, &r.Item, &r.Class, &r.Path, &r.FirstSeen, &r.DeclaredAt, &r.Size, + &r.LastWrite, &r.MeasuredAt, &r.LastBackup, &r.RetiredAt, &r.RetiredWhy, &r.DeletedAt, &r.DeletedBy, + &r.DeletedWhy, &r.Owned, &r.Protection, &r.MeasureError, &kind, &where, &healthy, &said, &r.Precision) + if err == nil && kind != nil { + r.Redundancy = &Redundancy{Kind: *kind, Healthy: healthy} + if where != nil { + r.Redundancy.Where = *where + } + if said != nil { + r.Redundancy.Said = *said + } + } + return r, err +} + +func nullable(s string) *string { + if s == "" { + return nil + } + return &s +} + +// Data is every item the mesh keeps, by machine, module and item. +func (i *Inventory) Data(ctx context.Context) ([]DataRecord, error) { + rows, err := i.store.Pool().Query(ctx, dataSelect+` order by machine, module, item`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []DataRecord + for rows.Next() { + r, err := scanData(rows) + if err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +// DataOf is one item. +func (i *Inventory) DataOf(ctx context.Context, machine, module, item string) (DataRecord, error) { + r, err := scanData(i.store.Pool().QueryRow(ctx, dataSelect+` where machine = $1 and module = $2 and item = $3`, + machine, module, item)) + if errors.Is(err, pgx.ErrNoRows) { + return r, fmt.Errorf("the mesh knows no data %s of %s on %s", item, module, machine) + } + return r, err +} + +// DataPeaks is each item's largest reading since a moment, keyed by DataRecord.Key. +func (i *Inventory) DataPeaks(ctx context.Context, since time.Time) (map[string]int64, error) { + rows, err := i.store.Pool().Query(ctx, `select machine, module, item, max(size_bytes) from data_reading + where at >= $1 group by machine, module, item`, since) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]int64{} + for rows.Next() { + var machine, module, item string + var peak int64 + if err := rows.Scan(&machine, &module, &item, &peak); err != nil { + return nil, err + } + out[machine+"/"+module+"/"+item] = peak + } + return out, rows.Err() +} + +// MarkDataDeleted records that a person deleted a retired item. Refused for an item not retired. +func (i *Inventory) MarkDataDeleted(ctx context.Context, machine, module, item, by, why string, at time.Time) error { + tag, err := i.store.Pool().Exec(ctx, `update data_item set deleted_at = $4, deleted_by = $5, deleted_why = $6 + where machine = $1 and module = $2 and item = $3 and retired_at is not null and deleted_at is null`, + machine, module, item, at, by, why) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + return fmt.Errorf("%s of %s on %s is not retired: only retired data is deleted", item, module, machine) + } + return nil +} diff --git a/internal/inventory/data_test.go b/internal/inventory/data_test.go new file mode 100644 index 0000000..d109c1b --- /dev/null +++ b/internal/inventory/data_test.go @@ -0,0 +1,130 @@ +package inventory + +import ( + "testing" + "time" +) + +func size(n int64) *int64 { return &n } + +func at(t time.Time) *time.Time { return &t } + +// What a machine declares is kept with what its holder measured; an irreplaceable item it no longer +// declares — its module unassigned — is RETIRED and kept, never forgotten, and comes back as it was when +// declared again; anything else no longer declared is forgotten (novox/hq ADR 0233). +func TestAnUndeclaredIrreplaceableItemIsRetiredNotForgotten(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) + declared := []DeclaredData{ + {Module: "home-assistant", Item: "config", Class: "irreplaceable", Owned: true}, + {Module: "searxng", Item: "valkey", Class: "cache"}, + {Module: "ollama", Item: "models", Class: "rebuildable"}, + } + measured := map[string]map[string]Measurement{"home-assistant": {"config": {Path: "/var/lib/home-assistant/config", + Size: size(900 << 20), LastWrite: at(now.Add(-time.Minute)), MeasuredAt: at(now), LastBackup: at(now.Add(-6 * time.Hour))}}} + if _, err := inv.RecordData(ctx, "home", declared, measured, "", now); err != nil { + t.Fatal(err) + } + all, err := inv.Data(ctx) + if err != nil || len(all) != 3 { + t.Fatalf("%+v, %v", all, err) + } + + // Unassigned: nothing of it is declared any more. + later := now.Add(time.Hour) + change, err := inv.RecordData(ctx, "home", nil, nil, "home-assistant unassigned", later) + if err != nil { + t.Fatal(err) + } + if len(change.Retired) != 1 || change.Retired[0].Item != "config" { + t.Fatalf("retired %+v", change.Retired) + } + all, err = inv.Data(ctx) + if err != nil { + t.Fatal(err) + } + if len(all) != 1 || !all[0].Retired() || all[0].Path != "/var/lib/home-assistant/config" || + *all[0].Size != 900<<20 || all[0].RetiredWhy != "home-assistant unassigned" { + t.Fatalf("the irreplaceable item is not kept retired with where it is: %+v", all) + } + + // A second run that still does not declare it changes nothing: retired once, not again. + change, err = inv.RecordData(ctx, "home", nil, nil, "again", later.Add(time.Hour)) + if err != nil || len(change.Retired) != 0 { + t.Fatalf("retired twice: %+v, %v", change, err) + } + + // Deleting needs it retired; assigned again it is not retired any more, and its history stays. + if err := inv.MarkDataDeleted(ctx, "home", "searxng", "valkey", "p", "w", later); err == nil { + t.Fatal("deleting something the mesh does not hold retired was recorded") + } + change, err = inv.RecordData(ctx, "home", declared[:1], nil, "", later.Add(2*time.Hour)) + if err != nil || len(change.Reenabled) != 1 { + t.Fatalf("declared again: %+v, %v", change, err) + } + r, err := inv.DataOf(ctx, "home", "home-assistant", "config") + if err != nil || r.Retired() || !r.FirstSeen.Equal(now) || r.Size == nil { + t.Fatalf("declared again lost what was known: %+v, %v", r, err) + } + + // Retired and then deleted by a person: recorded, never by dropping the row. + if _, err := inv.RecordData(ctx, "home", nil, nil, "unassigned again", later.Add(3*time.Hour)); err != nil { + t.Fatal(err) + } + if err := inv.MarkDataDeleted(ctx, "home", "home-assistant", "config", "jochen", "moved to the anchor", later.Add(4*time.Hour)); err != nil { + t.Fatal(err) + } + r, err = inv.DataOf(ctx, "home", "home-assistant", "config") + if err != nil || r.DeletedAt == nil || r.DeletedBy != "jochen" || r.Retired() { + t.Fatalf("a deletion is not on record: %+v, %v", r, err) + } +} + +// A reading is kept at most once an hour, and the peak is read over the window asked. +func TestReadingsAreHourlyAndThePeakIsTheLargest(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + now := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC) + declared := []DeclaredData{{Module: "grafana", Item: "data", Class: "valuable", Owned: true}} + for i, s := range []int64{100, 120, 999, 130, 40} { + when := now.Add(time.Duration(i) * 20 * time.Minute) + if _, err := inv.RecordData(ctx, "ace", declared, map[string]map[string]Measurement{"grafana": {"data": { + Path: "/var/lib/grafana/data", Size: size(s), MeasuredAt: at(when)}}}, "", when); err != nil { + t.Fatal(err) + } + } + var n int + if err := inv.store.Pool().QueryRow(ctx, `select count(*) from data_reading`).Scan(&n); err != nil || n != 2 { + t.Fatalf("%d readings kept for five measurements over 80 minutes, want 2 (one an hour): %v", n, err) + } + peaks, err := inv.DataPeaks(ctx, now.Add(-time.Hour)) + if err != nil || peaks["ace/grafana/data"] != 130 { + t.Fatalf("peak %v, %v", peaks, err) + } + r, _ := inv.DataOf(ctx, "ace", "grafana", "data") + if r.Size == nil || *r.Size != 40 { + t.Fatalf("the newest measurement is not the one on record: %+v", r) + } +} + +// A partial walk's lower bound is kept as the newest measurement, said as partial, and never kept as a +// reading a shrink would be read against. +func TestAPartialMeasurementIsNeverAReading(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + now := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC) + declared := []DeclaredData{{Module: "big", Item: "data", Class: "valuable", Owned: true}} + if _, err := inv.RecordData(ctx, "home", declared, map[string]map[string]Measurement{"big": {"data": { + Path: "/srv/big", Size: size(5), MeasuredAt: at(now), Precision: "partial: stopped"}}}, "", now); err != nil { + t.Fatal(err) + } + var n int + if err := inv.store.Pool().QueryRow(ctx, `select count(*) from data_reading`).Scan(&n); err != nil || n != 0 { + t.Fatalf("%d readings from a partial measurement: %v", n, err) + } + r, err := inv.DataOf(ctx, "home", "big", "data") + if err != nil || r.Precision != "partial: stopped" || Comparable(r.Precision) { + t.Fatalf("%+v, %v", r, err) + } +} diff --git a/internal/inventory/migrations/0072-a-module-declares-the-data-it-holds.sql b/internal/inventory/migrations/0072-a-module-declares-the-data-it-holds.sql new file mode 100644 index 0000000..0e29e93 --- /dev/null +++ b/internal/inventory/migrations/0072-a-module-declares-the-data-it-holds.sql @@ -0,0 +1,68 @@ +-- The data a module declares, as the mesh found it on each machine (novox/hq ADR 0233). +-- +-- A module's manifest says what data it keeps and of which class. The self-check asks each machine's +-- backup holder what it measured of every declared item — its size, its last write, its last good +-- backup — and keeps that here: so a shrink is read against what the item held before, an item a +-- machine no longer declares is still known to be there, and an empty copy of an item is told from a +-- full one somewhere else. +-- +-- **Keyed by the machine's name, not a reference.** The data outlives the machine record, as a +-- binding's provider does (migration 0071): a machine leaving the mesh must not turn the record of +-- what it holds into nothing. +-- +-- **Retired, never removed.** An irreplaceable or valuable item in a module's own directory that its +-- machine no longer declares — its module unassigned — is marked retired, with when and why, and stays +-- until a person deletes it through `cleanup delete` (ADR 0230); the deletion is recorded here too, +-- never by dropping the row. An item on an operator's path (an access) is never the mesh's to retire. +-- +-- Numbered 0072, past 0071, the highest on main or any open branch when this was written. +create table data_item ( + machine text not null, + module text not null, + item text not null, + class text not null, + -- Whether it is in the module's own directory (the mesh's to retire) or an operator's path, and + -- how it is protected: backup, redundancy, both, or none. + owned boolean not null default true, + protection text not null default '', + -- Where it is on the machine, as the backup holder last said; empty until one has. + path text not null default '', + first_seen timestamptz not null default now(), + -- When a composition of the machine last declared it. + declared_at timestamptz not null default now(), + -- The newest measurement: size in bytes, the newest write inside it, and when it was measured. + size_bytes bigint, + last_write timestamptz, + measured_at timestamptz, + -- The newest good backup that covers it. + last_backup timestamptz, + -- What the holder could not measure, when it could not: the path gone, a walk refused. + measure_error text, + -- What the newest size is: exact, a dataset's whole size, partial (a lower bound) or none. + precision text, + -- The redundant storage it is on, as the holder read it: zfs, md or btrfs, which pool or device, + -- whether it is healthy, and what it said. + redundancy_kind text, + redundancy_where text, + redundancy_healthy boolean, + redundancy_said text, + retired_at timestamptz, + retired_why text, + deleted_at timestamptz, + deleted_by text, + deleted_why text, + primary key (machine, module, item) +); + +-- One row per measurement kept, at most one an hour per item, for ninety days: what a shrink is +-- read against. Only a comparable size is kept: exact, or a dataset's own counters — never a partial +-- walk's lower bound. +create table data_reading ( + machine text not null, + module text not null, + item text not null, + at timestamptz not null, + size_bytes bigint not null, + last_write timestamptz, + primary key (machine, module, item, at) +); diff --git a/internal/link/retirement.go b/internal/link/retirement.go index fe6b532..2f580c1 100644 --- a/internal/link/retirement.go +++ b/internal/link/retirement.go @@ -233,10 +233,13 @@ type RetirementRejected struct { // RetirementState is a provider's answer to provisioner_retirement. type RetirementState struct { - Resource string `json:"resource"` - Node string `json:"node"` - Held []string `json:"held"` - StablePasses int `json:"stable_passes"` + Resource string `json:"resource"` + Node string `json:"node"` + Held []string `json:"held"` + // HeldSizes is what each held consumer keeps on the backend, in bytes, where the backend can say + // (novox/hq ADR 0233): what an empty replacement of a consumer's data is told by. + HeldSizes map[string]int64 `json:"held_sizes,omitempty"` + StablePasses int `json:"stable_passes"` // StableForSeconds is how long the same unasked set must hold as well (ADR 0230: ten minutes). StableForSeconds int `json:"stable_for_seconds,omitempty"` // RetiresBy is "disable", or "mark-only" for a provider that cannot disable a consumer.