Merge pull request 'A module declares the data it holds; protection and D13 derived from it (hq ADR 0233)' (#88) from feat/a-module-declares-the-data-it-holds into main
mesh/delivery held for a person: merged without a passing check: only a person decides that it goes on

This commit was merged in pull request #88.
This commit is contained in:
2026-10-06 15:00:53 +00:00
26 changed files with 3094 additions and 130 deletions
+4
View File
@@ -224,6 +224,10 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
for _, line := range unheldChange(shelf, node, assigned, left) { for _, line := range unheldChange(shelf, node, assigned, left) {
answer += "\n " + line answer += "\n " + line
} }
// What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233).
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
answer += "\n " + line
}
return answer + blockedElsewhere(ctx, open, node), nil return answer + blockedElsewhere(ctx, open, node), nil
} }
+17
View File
@@ -93,6 +93,15 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
} }
failed += len(identities) failed += len(identities)
// And the data each module keeps (novox/hq ADR 0233): a provider that grants says what it keeps for
// its consumers, a directory a container writes is declared, and no backup line is written by hand.
data := catalogue.DataProblems(shelf)
sort.Strings(data)
for _, p := range data {
fmt.Fprintln(out, p)
}
failed += len(data)
var names []string var names []string
for name := range shelf { for name := range shelf {
names = append(names, name) names = append(names, name)
@@ -121,6 +130,14 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
if len(m.Reads) > 0 { if len(m.Reads) > 0 {
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", ")) fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
} }
// The data it keeps, by class, so a reviewer sees what the mesh will protect and how.
if items := m.DataItems(); len(items) > 0 {
kept := make([]string, 0, len(items))
for _, it := range items {
kept = append(kept, it.ID+" ("+it.Class+")")
}
fmt.Fprintf(out, ", keeps %s", strings.Join(kept, ", "))
}
fmt.Fprintln(out) fmt.Fprintln(out)
} }
if failed > 0 { if failed > 0 {
+918
View File
@@ -0,0 +1,918 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"log"
"sort"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A module declares the data it holds, and the mesh protects and watches it from that declaration
// (novox/hq ADR 0233).
//
// The self-check's D13 composes what every machine declares, asks each machine's backup holder what
// it measured of every item — size, newest write, newest good backup, the redundant storage it is on —
// and keeps both. From that, and from what every provider says it holds for its consumers, it raises,
// each URGENT for what is irreplaceable and a WARNING for what is valuable (the operator's ranking):
//
// - `data-shrank`: an item holds less than half of its largest size in seven days, and at least
// shrinkFloor less; `data-missing`: its path is gone;
// - `empty-replacement`: an item, or a consumer's data at a provider, is less than half the size of a
// copy of the same thing kept elsewhere — on 2026-10-05 five applications ran for twenty hours on
// empty databases while their real ones sat on another machine (issue 273);
// - `data-held-twice` (warning): a consumer has active data at two providers and their sizes cannot
// be compared;
// - `data-quiet`: an item said to be written all the time has not been, within its bound;
// - `backup-stale`: an item's newest good backup is older than its bound, or there is none;
// - `array-degraded`: the redundant storage an item is on is not healthy, or cannot be read;
// `protection-missing`: an item said to be protected by redundancy is on storage that is not;
// - `cleanup-waiting` (warning): an item retired more than thirty days, waiting for a person.
//
// And it retires: an irreplaceable or valuable item in a module's own directory that its machine no
// longer declares — its module unassigned — is kept, marked retired with when and why, and listed by
// `cleanup list` until `cleanup delete` removes it. The node-engine never deletes a directory with
// anything in it; this is the record of what it kept. An operator's path is never retired or deleted.
// The condition kinds of D13.
const (
kindDataShrank = "data-shrank"
kindEmptyReplacement = "empty-replacement"
kindDataHeldTwice = "data-held-twice"
kindDataQuiet = "data-quiet"
kindBackupStale = "backup-stale"
kindDataUnmeasured = "data-unmeasured"
// kindDataMissing is a watched item whose path is gone.
kindDataMissing = "data-missing"
// kindArrayDegraded is redundant storage watched data is on that is not healthy, or cannot be read.
kindArrayDegraded = "array-degraded"
// kindProtectionMissing is an item said to be protected by redundancy, on storage that is not.
kindProtectionMissing = "protection-missing"
)
// probeDataID is the self-check's id for this probe.
const probeDataID = "D13"
// The bounds the findings are read against.
var (
// shrinkWindow is how far back the largest size is looked for.
shrinkWindow = 7 * 24 * time.Hour
// shrinkFloor is the least loss that is worth saying: two empty databases differ by a few
// megabytes, and half of almost nothing is noise.
shrinkFloor int64 = 16 << 20
// dataAsk is how long one machine's holder, or one provider, is given to answer.
dataAsk = 8 * time.Second
)
// keyOfItem is one item's condition id: its machine, module and item.
func keyOfItem(machine, module, item string) string { return machine + "." + module + "." + item }
// holderAnswer is what a node-backup holder's `backed-up` says of one module (ADR 0233 adds Data).
type holderAnswer struct {
Module string `json:"module"`
Data []holderItem `json:"data"`
}
// holderItem is one item as the holder measured it.
type holderItem struct {
Item string `json:"item"`
Class string `json:"class"`
Path string `json:"path"`
SizeBytes *int64 `json:"size_bytes"`
LastWrite *time.Time `json:"last_write"`
MeasuredAt *time.Time `json:"measured_at"`
LastBackup *time.Time `json:"last_backup"`
Error string `json:"error,omitempty"`
// Precision is what the size is: exact, a dataset's, partial, or none (ADR 0233).
Precision string `json:"precision,omitempty"`
// Redundancy is the redundant storage the item is on, where the holder could tell (ADR 0233).
Redundancy *inventory.Redundancy `json:"redundancy,omitempty"`
}
// readHolder reads a holder's answer into measurements by module and item.
func readHolder(raw json.RawMessage) (map[string]map[string]inventory.Measurement, error) {
var modules []holderAnswer
if err := json.Unmarshal(raw, &modules); err != nil {
return nil, fmt.Errorf("its answer is not readable: %w", err)
}
out := map[string]map[string]inventory.Measurement{}
for _, m := range modules {
for _, it := range m.Data {
if out[m.Module] == nil {
out[m.Module] = map[string]inventory.Measurement{}
}
out[m.Module][it.Item] = inventory.Measurement{Path: it.Path, Size: it.SizeBytes, LastWrite: it.LastWrite,
MeasuredAt: it.MeasuredAt, LastBackup: it.LastBackup, Error: it.Error, Redundancy: it.Redundancy,
Precision: it.Precision}
}
}
return out, nil
}
// declaredOn is every data item a machine's composition declares, and whether something there holds
// node-backup to measure them.
func declaredOn(plan catalogue.Resolution) ([]inventory.DeclaredData, bool) {
var out []inventory.DeclaredData
held := false
for _, m := range plan.Modules {
for _, c := range m.Claims {
if s, known := catalogue.SeatNamed(c.Name); known && s.Name == catalogue.BackupSeat {
held = true
}
}
for _, it := range m.DataItems() {
out = append(out, inventory.DeclaredData{Module: m.Module, Item: it.ID, Class: it.Class,
Owned: it.OwnedByModule(), Protection: it.Protection()})
}
}
return out, held
}
// consumerCopy is one provider's account of one consumer: where, how big, and whether still active.
type consumerCopy struct {
Node, Module, Consumer string
Size *int64
Retired bool
// Class is how precious the consumer's data is: the stricter of what the provider keeps for its
// consumers and what the consumer says it keeps there (`kept-by`).
Class string
}
// probeData is D13.
func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
if d.js == nil {
return nil, errors.New("no bus to ask the machines over")
}
open := d.open
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return nil, err
}
nodes, err := open.inventory.Nodes(ctx)
if err != nil {
return nil, err
}
heard := heardMachines(d)
now := time.Now()
type machine struct {
name string
declared []inventory.DeclaredData
held bool
measured map[string]map[string]inventory.Measurement
askErr error
}
var machines []*machine
for _, n := range nodes {
plan, _, err := planFor(ctx, open, n.Name)
if err != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
}
// A machine that cannot be worked out declares nothing this run — which is not the same as
// declaring nothing: retiring its data on that would be acting on an unreadable result.
continue
}
declared, held := declaredOn(plan)
machines = append(machines, &machine{name: n.Name, declared: declared, held: held})
}
// Every holder asked at once, as D8 asks every ban list.
var wg sync.WaitGroup
for _, m := range machines {
if !m.held || !heard[m.name] {
continue
}
wg.Add(1)
go func(m *machine) {
defer wg.Done()
asking, cancel := context.WithTimeout(ctx, dataAsk)
defer cancel()
raw, err := askSeatTool(asking, d.js.Conn(), catalogue.BackupSeat, "backed-up", m.name)
if err == nil {
m.measured, err = readHolder(raw)
}
m.askErr = err
}(m)
}
wg.Wait()
var out []conditions.Observation
for _, m := range machines {
if m.askErr != nil {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Token: kindDataUnmeasured,
Kind: kindDataUnmeasured, Machine: m.name, Severity: conditions.Warning,
Summary: fmt.Sprintf("%s's backup holder did not say what it measured of the data declared there, so "+
"nothing about that data is known this run: %s", m.name, firstLine(m.askErr.Error())),
Said: firstLine(m.askErr.Error())})
}
why := fmt.Sprintf("no longer declared on %s: its module was unassigned there, or is no longer pulled in", m.name)
change, err := open.inventory.RecordData(ctx, m.name, m.declared, m.measured, why, now)
if err != nil {
return nil, fmt.Errorf("what %s holds could not be kept: %w", m.name, err)
}
for _, r := range change.Retired {
log.Printf("data: %s of %s on %s RETIRED, kept at %s: %s — `cleanup list` shows it, and only `cleanup "+
"delete` removes it (novox/hq ADR 0233)", r.Item, r.Module, r.Machine, orUnknownPath(r.Path), why)
}
for _, r := range change.Reenabled {
log.Printf("data: %s of %s on %s is declared again, no longer retired", r.Item, r.Module, r.Machine)
}
}
records, err := open.inventory.Data(ctx)
if err != nil {
return nil, err
}
peaks, err := open.inventory.DataPeaks(ctx, now.Add(-shrinkWindow))
if err != nil {
return nil, err
}
bindings, err := open.inventory.Bindings(ctx)
if err != nil {
return nil, err
}
upgraded, keptBy := keptByClasses(bindings, shelf)
copies, err := consumerCopies(ctx, d.js.Conn(), open.inventory, shelf, keptBy)
if err != nil {
return nil, err
}
out = append(out, dataFindings(records, peaks, shelf, copies, upgraded, now)...)
return out, nil
}
func orUnknownPath(p string) string {
if p == "" {
return "a path its backup holder never named"
}
return p
}
// consumerCopies asks every provider of a provision whose consumers' data is kept what it holds, at
// once. One that cannot answer is passed over: it says nothing about any copy, which is not a finding.
func consumerCopies(ctx context.Context, conn *nats.Conn, inv *inventory.Inventory,
shelf map[string]catalogue.Manifest, keptBy map[string]string) ([]consumerCopy, error) {
instances, err := providerInstances(ctx, inv)
if err != nil {
return nil, err
}
var asked []providerInstance
for _, p := range instances {
m := shelf[p.Module]
keeps := false
for provision := range m.Grants {
keeps = keeps || m.KeepsConsumerData(provision)
}
if keeps {
asked = append(asked, p)
}
}
states := make([]*link.RetirementState, len(asked))
var wg sync.WaitGroup
for i, p := range asked {
wg.Add(1)
go func(i int, p providerInstance) {
defer wg.Done()
asking, cancel := context.WithTimeout(ctx, dataAsk)
defer cancel()
if s, err := askRetirement(asking, conn, p); err == nil {
states[i] = &s
}
}(i, p)
}
wg.Wait()
var out []consumerCopy
for i, p := range asked {
s := states[i]
if s == nil {
continue
}
class := consumersClass(shelf[p.Module])
for _, c := range s.Held {
cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: c,
Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+c])}
if size, ok := s.HeldSizes[c]; ok && size >= 0 {
size := size
cp.Size = &size
}
out = append(out, cp)
}
for _, r := range s.Retired {
if r.Kind != "" && r.Kind != "consumer" {
continue
}
cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: r.Consumer, Retired: true,
Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+r.Consumer])}
if r.SizeBytes != nil && *r.SizeBytes >= 0 {
cp.Size = r.SizeBytes
}
out = append(out, cp)
}
}
return out, nil
}
// severityOf is how loud a finding about data of a class is: urgent for what is irreplaceable, a warning
// for anything else watched (the operator's ranking, ADR 0233).
func severityOf(class string) conditions.Severity {
if class == catalogue.ClassIrreplaceable {
return conditions.Urgent
}
return conditions.Warning
}
// consumersClass is the most precious class a provider keeps any of its consumers' data as.
func consumersClass(m catalogue.Manifest) string {
class := catalogue.ClassNone
for provision := range m.Grants {
if c, ok := m.ConsumerDataOf(provision); ok {
class = catalogue.StricterClass(class, c.Class)
} else if m.KeepsConsumerData(provision) {
class = catalogue.StricterClass(class, catalogue.ClassValuable)
}
}
return class
}
// keptByClasses is what consumers say of the data they keep with their providers (`kept-by`), read
// through where each is bound: by provider module and consumer identity, the class of that consumer's
// data there; and by provider item key (machine/module/item), the class the item holding it is held to.
func keptByClasses(bindings []inventory.Binding, shelf map[string]catalogue.Manifest) (map[string]string, map[string]string) {
upgraded, keptBy := map[string]string{}, map[string]string{}
for _, b := range bindings {
m, ok := shelf[b.Consumer]
if !ok {
continue
}
k, said := m.KeptByOf(b.Provision)
if !said {
continue
}
identity := catalogue.ConsumerIdentity(b.Machine, catalogue.IdentitySource(m.Slug, m.Module))
key := b.Provider.Module + "/" + identity
keptBy[key] = catalogue.StricterClass(keptBy[key], k.Class)
if pc, ok := shelf[b.Provider.Module].ConsumerDataOf(b.Provision); ok && pc.In != "" {
if _, own := shelf[b.Provider.Module].DataItem(pc.In); own {
item := b.Provider.Node + "/" + b.Provider.Module + "/" + pc.In
upgraded[item] = catalogue.StricterClass(upgraded[item], k.Class)
}
}
}
return upgraded, keptBy
}
// dataFindings is every condition the data on record raises now. A function of what is known, so the
// incident's shape is tested without a mesh. upgraded is the class an item is held to where a consumer
// of its module keeps data in it more precious than its own class says (`kept-by`), by its key.
func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf map[string]catalogue.Manifest,
copies []consumerCopy, upgraded map[string]string, now time.Time) []conditions.Observation {
var out []conditions.Observation
byItem := map[string][]inventory.DataRecord{}
arrays := map[string][]inventory.DataRecord{}
type shrunk struct {
machine, dataset, class string
size, peak int64
items []string
}
shrunkDatasets := map[string]shrunk{}
for _, r := range records {
if r.DeletedAt != nil {
continue
}
class := catalogue.StricterClass(r.Class, upgraded[r.Key()])
r.Class = class
byItem[r.Module+"/"+r.Item] = append(byItem[r.Module+"/"+r.Item], r)
item, declared := shelf[r.Module].DataItem(r.Item)
id := keyOfItem(r.Machine, r.Module, r.Item)
if r.Retired() {
if now.Sub(*r.RetiredAt) > cleanupAfter {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "cleanup",
Kind: kindCleanupWaiting, Machine: r.Machine, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept at %s since %s; `cleanup "+
"delete %s %s %s --why …` once a person has decided, or assign %s there again",
r.Item, r.Module, r.Machine, r.Class, sizeWords(r.Size), int(now.Sub(*r.RetiredAt).Hours()/24),
orUnknownPath(r.Path), r.RetiredWhy, r.Machine, r.Module, r.Item, r.Module)})
}
continue
}
if !catalogue.Watched(class) {
continue
}
severity := severityOf(class)
if r.Redundancy != nil {
where := r.Machine + "/" + r.Redundancy.Kind + ":" + r.Redundancy.Where
arrays[where] = append(arrays[where], r)
} else if declared && item.Redundancy != "" && r.MeasuredAt != nil && r.MeasureError == "" {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindProtectionMissing,
Kind: kindProtectionMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s of %s on %s (%s) is said to be protected by the redundancy of the storage it is on, "+
"and %s is on nothing the backup holder can read as redundant: it has no protection the mesh can see",
r.Item, r.Module, r.Machine, class, orUnknownPath(r.Path))})
}
if r.MeasureError != "" && strings.Contains(r.MeasureError, "does not exist") {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataMissing,
Kind: kindDataMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s of %s on %s (%s) is gone: %s does not exist any more", r.Item, r.Module, r.Machine,
class, orUnknownPath(r.Path))})
} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) &&
*r.Size*2 < peak && peak-*r.Size >= shrinkFloor && inventory.Dataset(r.Precision) != "" {
// Several items on one dataset share its size: one condition for the dataset, as loud as the
// most precious item on it.
k := r.Machine + "/" + inventory.Dataset(r.Precision)
ds := shrunkDatasets[k]
ds.machine, ds.dataset, ds.size, ds.peak = r.Machine, inventory.Dataset(r.Precision), *r.Size, peak
ds.class = catalogue.StricterClass(ds.class, class)
ds.items = append(ds.items, r.Module+"/"+r.Item)
shrunkDatasets[k] = ds
} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) &&
*r.Size*2 < peak && peak-*r.Size >= shrinkFloor {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataShrank,
Kind: kindDataShrank, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s of %s on %s (%s) shrank to %s from %s within %d days — more than half of what it "+
"held is gone. If that was meant, silence this with why; if not, `node-backup.restore` puts the last "+
"good copy beside it", r.Item, r.Module, r.Machine, class, sizeWords(r.Size), sizeWords(&peak),
int(shrinkWindow.Hours()/24))})
}
if !declared {
continue
}
if within := item.ActiveWithin(); within > 0 && r.LastWrite != nil && now.Sub(*r.LastWrite) > within {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataQuiet,
Kind: kindDataQuiet, Machine: r.Machine, Severity: severity,
Summary: fmt.Sprintf("%s of %s on %s is written all the time, and has not been since %s (its bound is %s): "+
"whatever writes it has stopped", r.Item, r.Module, r.Machine, r.LastWrite.UTC().Format(time.RFC3339),
within)})
}
// A backup is required of what is irreplaceable and copied; of what is valuable it is the standard
// plan, said only where the machine was measured — where a holder is there to take it.
if item.BackedUp() && (class == catalogue.ClassIrreplaceable || r.MeasuredAt != nil) {
within := item.BackupWithin()
switch {
case r.LastBackup == nil && now.Sub(r.FirstSeen) > within:
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale,
Kind: kindBackupStale, Machine: r.Machine, Severity: severity,
Summary: fmt.Sprintf("%s of %s on %s is %s and has no good backup on record, %s after it was first "+
"declared — is node-backup held there, and do its nights succeed? (`node-backup.backed-up`)",
r.Item, r.Module, r.Machine, class, now.Sub(r.FirstSeen).Round(time.Hour))})
case r.LastBackup != nil && now.Sub(*r.LastBackup) > within:
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale,
Kind: kindBackupStale, Machine: r.Machine, Severity: severity,
Summary: fmt.Sprintf("%s of %s on %s is %s and its newest good backup is from %s, older than its bound "+
"of %s", r.Item, r.Module, r.Machine, class, r.LastBackup.UTC().Format(time.RFC3339), within)})
}
}
}
for _, k := range keysSorted(shrunkDatasets) {
ds := shrunkDatasets[k]
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
ID: ds.machine + ".dataset." + strings.ReplaceAll(ds.dataset, "/", "-"), Token: kindDataShrank,
Kind: kindDataShrank, Machine: ds.machine, Severity: severityOf(ds.class), Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("the dataset %s on %s shrank to %s from %s within %d days — more than half of what it held "+
"is gone; it holds %s", ds.dataset, ds.machine, sizeWords(&ds.size), sizeWords(&ds.peak),
int(shrinkWindow.Hours()/24), strings.Join(ds.items, ", "))})
}
// The redundant storage watched data is on: one condition per array, as loud as the most precious
// item on it — the array, not each item, is what degrades.
for _, where := range keysSorted(arrays) {
rs := arrays[where]
red := rs[0].Redundancy
if red.Healthy != nil && *red.Healthy {
continue
}
class, machine := catalogue.ClassValuable, rs[0].Machine
var names []string
for _, r := range rs {
class = catalogue.StricterClass(class, r.Class)
names = append(names, r.Module+"/"+r.Item)
}
state := "could not be read"
if red.Healthy != nil {
state = "is NOT healthy"
}
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
ID: machine + ".array." + strings.NewReplacer("/", "-", ":", "-").Replace(red.Kind+"-"+red.Where),
Token: kindArrayDegraded, Kind: kindArrayDegraded, Machine: machine, Severity: severityOf(class),
Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("the %s storage %s on %s %s: %s — and it is what protects %s", red.Kind, red.Where, machine,
state, firstLine(red.Said), strings.Join(names, ", "))})
}
// The same item on several machines: a copy that is in use and far smaller than one kept elsewhere is
// an empty replacement. Only against a retired copy — a module running on two machines on purpose
// keeps two different sets of data.
for _, key := range keysSorted(byItem) {
rs := byItem[key]
for _, a := range rs {
if a.Retired() || a.Size == nil || !catalogue.Watched(a.Class) || !inventory.Comparable(a.Precision) {
continue
}
for _, o := range rs {
if o.Machine == a.Machine || !o.Retired() || o.Size == nil || !inventory.Comparable(o.Precision) ||
!replacedByLess(*a.Size, *o.Size) {
continue
}
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
ID: keyOfItem(a.Machine, a.Module, a.Item), Token: kindEmptyReplacement, Kind: kindEmptyReplacement,
Machine: a.Machine, Also: []string{o.Machine}, Severity: severityOf(a.Class), Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s of %s on %s holds %s, and the copy %s kept on %s holds %s: %s is running on "+
"an empty replacement of its data. Move the data, or assign it back where its data is",
a.Item, a.Module, a.Machine, sizeWords(a.Size), o.Module, o.Machine, sizeWords(o.Size), a.Module)})
break
}
}
}
out = append(out, consumerFindings(copies)...)
return out
}
// replacedByLess is whether a copy in use is an empty replacement of a copy kept elsewhere: less than
// half of it, and at least shrinkFloor less.
func replacedByLess(inUse, kept int64) bool {
return inUse*2 < kept && kept-inUse >= shrinkFloor
}
// consumerFindings is the same question of consumers' data at providers: one consumer, the same
// provider module on two machines.
func consumerFindings(copies []consumerCopy) []conditions.Observation {
by := map[string][]consumerCopy{}
for _, c := range copies {
k := c.Module + "/" + c.Consumer
by[k] = append(by[k], c)
}
var out []conditions.Observation
for _, k := range keysSorted(by) {
cs := by[k]
if len(cs) < 2 {
continue
}
found := false
for _, a := range cs {
if a.Retired || a.Size == nil {
continue
}
for _, o := range cs {
if o.Node == a.Node || o.Size == nil || !replacedByLess(*a.Size, *o.Size) {
continue
}
state := "active"
if o.Retired {
state = "retired"
}
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider,
ID: a.Module + "." + a.Node + "." + a.Consumer, Token: kindEmptyReplacement, Kind: kindEmptyReplacement,
Machine: a.Node, Also: []string{o.Node}, Severity: severityOf(catalogue.StricterClass(a.Class, o.Class)),
Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s's data at %s on %s holds %s, and its %s copy at %s on %s holds %s: the "+
"consumer is using an empty replacement of its data (issue 273's shape). Pin it back to %s, or move "+
"the data first", a.Consumer, a.Module, a.Node, sizeWords(a.Size), state, o.Module, o.Node,
sizeWords(o.Size), o.Node)})
found = true
break
}
if found {
break
}
}
if found {
continue
}
var active []consumerCopy
for _, c := range cs {
if !c.Retired {
active = append(active, c)
}
}
if len(active) >= 2 {
var where []string
var also []string
for _, c := range active {
where = append(where, c.Node+" ("+sizeWords(c.Size)+")")
also = append(also, c.Node)
}
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider,
ID: active[0].Module + "." + active[0].Consumer, Token: kindDataHeldTwice, Kind: kindDataHeldTwice,
Machine: active[0].Node, Also: also[1:], Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s has active data at %s on %d machines — %s — and only one is the one it uses",
active[0].Consumer, active[0].Module, len(active), strings.Join(where, ", "))})
}
}
return out
}
func keysSorted[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
// ---- the `data` verb ---------------------------------------------------------------------------
const dataUsage = "data [--json] [--machine <name>] [--retired]"
// dataRow is one item as `data` lists it.
type dataRow struct {
Machine string `json:"machine"`
Module string `json:"module"`
Item string `json:"item"`
Class string `json:"class"`
Path string `json:"path,omitempty"`
Protection string `json:"protection,omitempty"`
// Array is the redundant storage it is on and its state, where its holder could tell.
Array string `json:"array,omitempty"`
Unmeasured string `json:"unmeasured,omitempty"`
// Precision says what the size is: exact, a dataset's whole size, partial, or none.
Precision string `json:"precision,omitempty"`
SizeBytes *int64 `json:"size-bytes,omitempty"`
LastWrite string `json:"last-write,omitempty"`
MeasuredAt string `json:"measured-at,omitempty"`
LastBackup string `json:"last-backup,omitempty"`
BackupDue string `json:"backup-within,omitempty"`
Retired string `json:"retired,omitempty"`
RetiredWhy string `json:"retired-why,omitempty"`
Deleted string `json:"deleted,omitempty"`
}
// dataCommand is `data`: every item every machine declares, or held retired, as the self-check last
// found it.
func dataCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("data", flag.ContinueOnError)
asJSON := set.Bool("json", false, "as data")
only := set.String("machine", "", "one machine")
retiredOnly := set.Bool("retired", false, "only what is retired")
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New(dataUsage)
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
records, err := open.inventory.Data(ctx)
if err != nil {
return err
}
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return err
}
rows := dataRows(records, shelf, *only, *retiredOnly)
if *asJSON {
return printJSON(map[string]any{"data": rows})
}
if len(rows) == 0 {
fmt.Println("no data on record: the self-check (D13) records what each machine declares on its next run")
return nil
}
for _, r := range rows {
state := ""
switch {
case r.Deleted != "":
state = " DELETED " + r.Deleted
case r.Retired != "":
state = " RETIRED " + r.Retired + " — " + r.RetiredWhy
}
fmt.Printf("%s %s/%s %s %s %s protected by %s%s\n", r.Machine, r.Module, r.Item, r.Class,
sizeWords(r.SizeBytes), orUnknownPath(r.Path), orNothingWord(r.Protection), state)
if r.Array != "" {
fmt.Printf(" on %s\n", r.Array)
}
if r.Precision != "" && r.Precision != "exact" {
fmt.Printf(" size: %s\n", r.Precision)
}
if r.Unmeasured != "" {
fmt.Printf(" not measured: %s\n", r.Unmeasured)
}
if r.Class == catalogue.ClassCache {
continue
}
fmt.Printf(" last write %s, measured %s, last backup %s%s\n", orNever(r.LastWrite), orNever(r.MeasuredAt),
orNever(r.LastBackup), within(r.BackupDue))
}
return nil
}
func dataRows(records []inventory.DataRecord, shelf map[string]catalogue.Manifest, only string, retiredOnly bool) []dataRow {
rows := []dataRow{}
stamp := func(t *time.Time) string {
if t == nil {
return ""
}
return t.UTC().Format(time.RFC3339)
}
for _, r := range records {
if only != "" && r.Machine != only {
continue
}
if retiredOnly && !r.Retired() {
continue
}
row := dataRow{Machine: r.Machine, Module: r.Module, Item: r.Item, Class: r.Class, Path: r.Path,
Protection: r.Protection, Unmeasured: r.MeasureError, Precision: r.Precision, SizeBytes: r.Size, LastWrite: stamp(r.LastWrite), MeasuredAt: stamp(r.MeasuredAt),
LastBackup: stamp(r.LastBackup), Retired: stamp(r.RetiredAt), RetiredWhy: r.RetiredWhy,
Deleted: stamp(r.DeletedAt)}
if it, ok := shelf[r.Module].DataItem(r.Item); ok && it.BackedUp() {
row.BackupDue = it.BackupWithin().String()
}
if red := r.Redundancy; red != nil {
state := "state unread"
if red.Healthy != nil && *red.Healthy {
state = "healthy"
} else if red.Healthy != nil {
state = "NOT HEALTHY"
}
row.Array = red.Kind + " " + red.Where + ", " + state
}
rows = append(rows, row)
}
return rows
}
func orNothingWord(s string) string {
if s == "" || s == "none" {
return "nothing"
}
return s
}
func orNever(s string) string {
if s == "" {
return "never"
}
return s
}
func within(s string) string {
if s == "" {
return " (not backed up)"
}
return " (bound " + s + ")"
}
// ---- cleanup of retired own data ---------------------------------------------------------------
// The tools a node-backup holder serves to delete one retired item (novox/hq ADR 0233): the first
// takes a last restore point of it, tagged as retired, and only then removes it — in the background,
// because a large item outlasts any call — and the second says how that went. Module tools, not seat
// verbs: only the controller's `cleanup delete` calls them, as it calls a provider's provisioner_delete.
const (
ToolDeleteRetired = "backup_delete_retired"
ToolDeletedOutcome = "backup_deleted"
)
// deletionWait is how long `cleanup delete` follows a deletion before handing it back to the person.
var deletionWait = 8 * time.Minute
// deletionPoll is how often it asks.
var deletionPoll = 5 * time.Second
// deletion is a holder's account of one deletion.
type deletion struct {
Started bool `json:"started"`
Running bool `json:"running"`
Done bool `json:"done"`
OK bool `json:"ok"`
Snapshot string `json:"snapshot"`
Error string `json:"error"`
}
// retiredData is every retired item on record, as `cleanup list` shows them.
func retiredData(records []inventory.DataRecord, now time.Time) []retiredRow {
var out []retiredRow
for _, r := range records {
if !r.Retired() {
continue
}
out = append(out, retiredRow{Node: r.Machine, Module: r.Module, Consumer: r.Item, Kind: retiredDataKind,
RetiredAt: r.RetiredAt.UTC().Format(time.RFC3339), AgeDays: int(now.Sub(*r.RetiredAt).Hours() / 24),
SizeBytes: r.Size, Why: r.RetiredWhy, Path: r.Path, Class: r.Class})
}
return out
}
// retiredDataKind is what `cleanup list` calls a module's own retired data, beside a provider's consumer.
const retiredDataKind = "own-data"
// holderOn is the module holding node-backup on a machine.
func holderOn(ctx context.Context, inv *inventory.Inventory, machine string) (string, error) {
held, err := inv.Holdings(ctx)
if err != nil {
return "", err
}
for _, h := range held {
if s, known := catalogue.SeatNamed(h.Claim); known && s.Name == catalogue.BackupSeat && h.Node == machine {
return h.Module, nil
}
}
return "", fmt.Errorf("nothing holds %s on %s, and it is the backup holder that deletes retired data there "+
"(after a last restore point)", catalogue.BackupSeat, machine)
}
// deleteRetiredData has a machine's backup holder delete one retired item: never one declared now, and
// never one not retired. The holder takes a last restore point of it first, so the deletion can be
// undone until a person forgets that restore point; the record says deleted only once the holder says
// it is.
func deleteRetiredData(ctx context.Context, conn *nats.Conn, open *stores, r inventory.DataRecord, f handActFlags) error {
inv := open.inventory
if !r.Retired() {
return fmt.Errorf("%s of %s on %s is not retired — only retired data is deleted. Nothing was done",
r.Item, r.Module, r.Machine)
}
if r.Path == "" {
return fmt.Errorf("%s of %s on %s was never measured, so where it is was never said; nothing was deleted",
r.Item, r.Module, r.Machine)
}
if plan, _, err := planFor(ctx, open, r.Machine); err == nil {
for _, m := range plan.Modules {
if _, still := m.DataItem(r.Item); still && m.Module == r.Module {
return fmt.Errorf("%s runs on %s again and declares %s: it is not retired any more. Nothing was done",
r.Module, r.Machine, r.Item)
}
}
}
holder, err := holderOn(ctx, inv, r.Machine)
if err != nil {
return err
}
f.record(ctx, "cleanup delete", []string{r.Machine, r.Module, r.Item})
args := map[string]any{"module": r.Module, "item": r.Item, "path": r.Path, "confirm": r.Item,
"why": strings.TrimSpace(*f.why), "by": link.Caller(), "via": link.ViaController}
ask := func(tool string) (deletion, error) {
var d deletion
answer, err := link.AskModuleToolOn(ctx, conn, holder, tool, r.Machine, args, 25*time.Second)
if err != nil {
return d, err
}
if answer.Error != "" {
return d, fmt.Errorf("%s on %s refused: %s", holder, r.Machine, answer.Error)
}
return d, unmarshalAnswer(answer, &d)
}
d, err := ask(ToolDeleteRetired)
if err != nil {
return err
}
for waited := time.Duration(0); !d.Done && waited < deletionWait; waited += deletionPoll {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(deletionPoll):
}
if d, err = ask(ToolDeletedOutcome); err != nil {
return err
}
}
switch {
case !d.Done:
fmt.Printf("%s on %s is still taking the last restore point of %s and deleting it; `cleanup list` keeps "+
"showing it until the holder says it is done — the same `cleanup delete` again reads how it went\n",
holder, r.Machine, r.Path)
return nil
case !d.OK:
return fmt.Errorf("%s on %s did NOT delete %s: %s", holder, r.Machine, r.Path, d.Error)
}
if err := inv.MarkDataDeleted(ctx, r.Machine, r.Module, r.Item, link.Caller(), strings.TrimSpace(*f.why), time.Now()); err != nil {
return fmt.Errorf("%s deleted %s on %s, and it could not be recorded: %w", holder, r.Path, r.Machine, err)
}
fmt.Printf("%s on %s deleted %s of %s (%s, %s); its last restore point is %s, kept until a person forgets it\n",
holder, r.Machine, r.Item, r.Module, r.Path, sizeWords(r.Size), orNever(d.Snapshot))
return nil
}
// keptOnUnassign says, for an unassignment, the irreplaceable and valuable data each module leaves in its
// own directories on the machine:
// kept, and retired at the self-check's next run.
func keptOnUnassign(ctx context.Context, inv *inventory.Inventory, machine string, modules []string) []string {
records, err := inv.Data(ctx)
if err != nil {
return []string{"what it leaves behind could not be read from the mesh's record: " + err.Error()}
}
var out []string
for _, r := range records {
if r.Machine != machine || r.DeletedAt != nil || !catalogue.Retires(r.Class) || !r.Owned {
continue
}
for _, m := range modules {
if r.Module == m {
out = append(out, fmt.Sprintf("%s's %s (%s, %s) stays where it is: it is %s, so it is retired, "+
"never removed — `cleanup list` shows it, `cleanup delete` alone removes it (novox/hq ADR 0233)",
r.Module, r.Item, orUnknownPath(r.Path), sizeWords(r.Size), r.Class))
}
}
}
return out
}
+460
View File
@@ -0,0 +1,460 @@
package main
import (
"context"
"encoding/json"
"os"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
)
func bytesOf(n int64) *int64 { return &n }
func when(t time.Time) *time.Time { return &t }
func shelfFor(t *testing.T, manifests ...string) map[string]catalogue.Manifest {
t.Helper()
out := map[string]catalogue.Manifest{}
for _, raw := range manifests {
m, err := catalogue.ParseManifest([]byte(raw))
if err != nil {
t.Fatal(err)
}
out[m.Module] = m
}
return out
}
const houseManifest = `{"module":"house","version":"1",
"data":{"own":[{"id":"config","path":"${dir:config}","class":"irreplaceable","active":"1d"}]},
"resources":[{"id":"config","type":"directory","mode":"0700"}]}`
func findingsByKind(obs []conditions.Observation) map[string]conditions.Observation {
out := map[string]conditions.Observation{}
for _, o := range obs {
out[o.Kind] = o
}
return out
}
// THE INCIDENT (issue 273), replayed against what D13 reads: five applications on the home server bound,
// by one changed rule, to the store on the control node, which made each an empty database — while
// their real databases, hundreds of megabytes each, sat on the home server's own store, by then retired
// because the mesh no longer asked for them there. Each is an empty replacement, naming both machines
// and the pin back: a warning for the store's consumers, whose data is valuable; urgent for one that says
// its data there is irreplaceable (`kept-by`).
func TestAnEmptyReplacementOfAConsumersDataIsSaid(t *testing.T) {
var copies []consumerCopy
for _, app := range []string{"mesh_home_board", "mesh_home_flows", "mesh_home_agents", "mesh_home_game", "mesh_home_cars"} {
copies = append(copies,
consumerCopy{Node: "home", Module: "postgres", Consumer: app, Size: bytesOf(400 << 20), Retired: true, Class: "valuable"},
consumerCopy{Node: "anchor", Module: "postgres", Consumer: app, Size: bytesOf(9 << 20), Class: "valuable"})
}
copies[1].Class = "irreplaceable" // the photo site's own database says so
got := dataFindings(nil, nil, nil, copies, nil, time.Now())
if len(got) != 5 {
t.Fatalf("%d findings for five empty replacements: %+v", len(got), got)
}
for i, o := range got {
want := conditions.Warning
if strings.Contains(o.ID, "mesh_home_board") {
want = conditions.Urgent
}
_ = i
if o.Kind != kindEmptyReplacement || o.Severity != want || o.Machine != "anchor" ||
len(o.Also) != 1 || o.Also[0] != "home" || !strings.Contains(o.Summary, "Pin it back to home") {
t.Errorf("%+v", o)
}
}
// While the old copy is still active (the first ten minutes), it is the same finding.
copies[0].Retired = false
copies[1].Class = "valuable"
if got := dataFindings(nil, nil, nil, copies[:2], nil, time.Now()); len(got) != 1 || got[0].Kind != kindEmptyReplacement {
t.Fatalf("with the old copy still active: %+v", got)
}
}
// A move a person made — the data moved first, then pinned — leaves a full copy at the new provider and
// a retired one at the old: nothing to say here; `cleanup` covers the old one.
func TestADeliberateMoveIsNoEmptyReplacement(t *testing.T) {
copies := []consumerCopy{
{Node: "home", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(400 << 20), Retired: true},
{Node: "anchor", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(402 << 20)},
}
if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 {
t.Fatalf("a deliberate move raised %+v", got)
}
// Two small databases differing by less than the floor are not a finding either.
copies[0].Size, copies[1].Size = bytesOf(12<<20), bytesOf(8<<20)
if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 {
t.Fatalf("noise between two empty databases raised %+v", got)
}
}
// Where a provider cannot say sizes, a consumer active at two providers is still said — as a warning,
// since which one is empty cannot be told.
func TestConsumerDataActiveTwiceWithoutSizesIsAWarning(t *testing.T) {
copies := []consumerCopy{
{Node: "home", Module: "minio", Consumer: "mesh_home_photos"},
{Node: "anchor", Module: "minio", Consumer: "mesh_home_photos"},
}
got := dataFindings(nil, nil, nil, copies, nil, time.Now())
if len(got) != 1 || got[0].Kind != kindDataHeldTwice || got[0].Severity != conditions.Warning {
t.Fatalf("%+v", got)
}
}
// The same incident for a module's own data: a module unassigned from one machine and assigned on
// another starts over in an empty directory while its full one is kept, retired, where it was.
func TestAnEmptyReplacementOfAModulesOwnDataIsUrgent(t *testing.T) {
now := time.Now()
retired := now.Add(-time.Hour)
records := []inventory.DataRecord{
{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config",
Size: bytesOf(2 << 30), RetiredAt: &retired, FirstSeen: now.Add(-90 * 24 * time.Hour)},
{Machine: "anchor", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config",
Size: bytesOf(1 << 20), FirstSeen: now.Add(-time.Hour), LastWrite: when(now)},
}
got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))
o, ok := got[kindEmptyReplacement]
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "home" {
t.Fatalf("%+v", got)
}
// The same of a valuable item is a warning.
records[0].Class, records[1].Class = "valuable", "valuable"
if o := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))[kindEmptyReplacement]; o.Severity != conditions.Warning {
t.Fatalf("a valuable empty replacement: %+v", o)
}
records[0].Class, records[1].Class = "irreplaceable", "irreplaceable"
// Two machines running a module on purpose, both active, keep two sets of data: nothing to say.
records[0].RetiredAt = nil
if got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now)); got[kindEmptyReplacement].Kind != "" {
t.Fatalf("two active copies were read as a replacement: %+v", got)
}
}
// An irreplaceable item that lost more than half of its largest size in a week is urgent; a smaller loss,
// or a loss under the floor, is not a finding.
func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) {
now := time.Now()
r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
Size: bytesOf(300 << 20), FirstSeen: now.Add(-30 * 24 * time.Hour), LastWrite: when(now), LastBackup: when(now)}
shelf := shelfFor(t, houseManifest)
o := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): 1 << 30}, shelf, nil, nil, now))[kindDataShrank]
if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "shrank") {
t.Fatalf("%+v", o)
}
for _, peak := range []int64{500 << 20, 20 << 20} {
if got := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): peak}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" {
t.Errorf("a peak of %d raised a shrink", peak)
}
}
small := r
small.Size = bytesOf(1 << 20)
if got := findingsByKind(dataFindings([]inventory.DataRecord{small}, map[string]int64{r.Key(): 10 << 20}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" {
t.Error("a loss under the floor raised a shrink")
}
}
// Data said to be written all the time and not written; data with no backup or an old one — urgent when
// irreplaceable, a warning when valuable; and a new item given its bound before it is said.
func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) {
now := time.Now()
shelf := shelfFor(t, houseManifest)
r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
Size: bytesOf(1 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), LastWrite: when(now.Add(-3 * 24 * time.Hour)),
LastBackup: when(now.Add(-72 * time.Hour))}
got := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now))
if got[kindDataQuiet].Severity != conditions.Urgent || got[kindBackupStale].Severity != conditions.Urgent {
t.Fatalf("irreplaceable: %+v", got)
}
valuable := r
valuable.Class, valuable.MeasuredAt = "valuable", when(now) // measured: a holder is there to take its backup
if got := findingsByKind(dataFindings([]inventory.DataRecord{valuable}, nil, shelf, nil, nil, now)); got[kindDataQuiet].Severity != conditions.Warning ||
got[kindBackupStale].Severity != conditions.Warning {
t.Fatalf("valuable: %+v", got)
}
never := r
never.LastBackup = nil
if o := findingsByKind(dataFindings([]inventory.DataRecord{never}, nil, shelf, nil, nil, now))[kindBackupStale]; !strings.Contains(o.Summary, "no good backup") {
t.Fatalf("never backed up: %+v", o)
}
fresh := never
fresh.FirstSeen, fresh.LastWrite = now.Add(-time.Hour), when(now)
if got := dataFindings([]inventory.DataRecord{fresh}, nil, shelf, nil, nil, now); len(got) != 0 {
t.Fatalf("an item declared an hour ago, before its first night, raised %+v", got)
}
}
// An item retired more than thirty days waits for a person; less, it is only listed.
func TestRetiredDataWaitingThirtyDaysIsSaid(t *testing.T) {
now := time.Now()
old, recent := now.Add(-31*24*time.Hour), now.Add(-2*24*time.Hour)
records := []inventory.DataRecord{
{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &old},
{Machine: "home", Module: "attic", Item: "boxes", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &recent},
}
got := dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now)
if len(got) != 1 || got[0].Kind != kindCleanupWaiting || !strings.Contains(got[0].Summary, "cleanup delete home house config") {
t.Fatalf("%+v", got)
}
if rows := retiredData(records, now); len(rows) != 2 || rows[0].Kind != retiredDataKind {
t.Fatalf("cleanup list: %+v", rows)
}
}
// The holder's answer reads into measurements, by module and item.
func TestTheHoldersAnswerIsRead(t *testing.T) {
raw := []byte(`[{"module":"postgres","runs":1,"paths":["/var/lib/mesh-store/dumps"],"lastNight":null,"restorePoints":3,
"data":[{"item":"store","class":"irreplaceable","path":"/var/lib/mesh-store","covered_by":"/var/lib/mesh-store/dumps",
"size_bytes":1073741824,"last_write":"2026-10-06T10:00:00Z","measured_at":"2026-10-06T10:05:00Z","last_backup":"2026-10-06T03:10:00Z"}]}]`)
got, err := readHolder(raw)
if err != nil {
t.Fatal(err)
}
m := got["postgres"]["store"]
if m.Path != "/var/lib/mesh-store" || m.Size == nil || *m.Size != 1<<30 || m.LastBackup == nil || m.MeasuredAt == nil {
t.Fatalf("%+v", m)
}
// An older holder, which says no data, reads as nothing measured rather than a failure.
if got, err := readHolder([]byte(`[{"module":"postgres","runs":1,"paths":[]}]`)); err != nil || len(got) != 0 {
t.Fatalf("%v, %v", got, err)
}
}
// fakeHolder answers node-backup's `backed-up` on one machine over a real bus, with what it is told it
// measured.
type fakeHolder struct {
mu sync.Mutex
modules []map[string]any
}
func (f *fakeHolder) set(modules ...map[string]any) {
f.mu.Lock()
defer f.mu.Unlock()
f.modules = modules
}
func (f *fakeHolder) serve(t *testing.T, conn *nats.Conn, node string) {
t.Helper()
sub, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) {
f.mu.Lock()
defer f.mu.Unlock()
body, _ := json.Marshal(map[string]any{"result": f.modules, "error": "", "node": node})
_ = m.Respond(body)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
if err := conn.Flush(); err != nil {
t.Fatal(err)
}
}
func measuredHouse(path string, size int64, at time.Time) map[string]any {
return map[string]any{"module": "house", "runs": 0, "paths": []string{path}, "data": []map[string]any{{
"item": "config", "class": "irreplaceable", "path": path, "covered_by": path, "size_bytes": size,
"last_write": at, "measured_at": at, "last_backup": at}}}
}
// UNASSIGNING A MODULE WITH IRREPLACEABLE DATA KEEPS THE DATA, and assigning it elsewhere onto an empty
// directory is an empty replacement — through the real stores and a real bus. The unassignment says the
// data stays; the self-check's next run retires it (kept, listed by cleanup), and when the module comes
// up on another machine with an empty directory while the full one waits retired, that is urgent.
func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
house, err := catalogue.ParseManifest([]byte(houseManifest))
if err != nil {
t.Fatal(err)
}
register(t, open, house)
if _, err := assign(ctx, open, "laptop", "house"); err == nil {
t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up")
}
for _, node := range []string{"laptop", "anchor"} {
if _, err := assign(ctx, open, node, "keeper"); err != nil {
t.Fatal(err)
}
}
if _, err := assign(ctx, open, "laptop", "house"); err != nil {
t.Fatal(err)
}
conn := onATestBus(t)
js, err := broker.Dial(os.Getenv("MESH_TEST_NATS"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
laptop, anchor := &fakeHolder{}, &fakeHolder{}
laptop.serve(t, conn, "laptop")
anchor.serve(t, conn, "anchor")
now := time.Now()
heard := &watchdogs{last: &signalFacts{now: now, machines: []machineFacts{
{name: "laptop", lastHeard: now}, {name: "anchor", lastHeard: now}}}}
d := &doctor{open: open, js: js, watchdogs: heard}
var d13 probe
for _, p := range probeRegistry {
if p.ID == probeDataID {
d13 = p
}
}
run := func() []conditions.Observation {
t.Helper()
probing := context.WithValue(ctx, probeAsksKey{}, d13)
obs, err := probeData(probing, d)
if err != nil {
t.Fatal(err)
}
return obs
}
laptop.set(measuredHouse("/var/lib/house/config", 2<<30, now))
if obs := run(); len(obs) != 0 {
t.Fatalf("a measured, backed-up item raised %+v", obs)
}
r, err := inv.DataOf(ctx, "laptop", "house", "config")
if err != nil || r.Path != "/var/lib/house/config" || r.Size == nil || *r.Size != 2<<30 || r.LastBackup == nil {
t.Fatalf("what the holder measured was not kept: %+v, %v", r, err)
}
said, err := unassign(ctx, open, "laptop", "house")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(said, "house's config (/var/lib/house/config, 2.0 GB) stays where it is") {
t.Fatalf("the unassignment does not say the data stays:\n%s", said)
}
laptop.set()
run()
r, err = inv.DataOf(ctx, "laptop", "house", "config")
if err != nil || !r.Retired() || r.Path != "/var/lib/house/config" {
t.Fatalf("unassigned, the irreplaceable item is not kept retired: %+v, %v", r, err)
}
records, _ := inv.Data(ctx)
if rows := retiredData(records, time.Now()); len(rows) != 1 || rows[0].Path != "/var/lib/house/config" {
t.Fatalf("cleanup list: %+v", rows)
}
// Assigned on the anchor, onto an empty directory.
if _, err := assign(ctx, open, "anchor", "house"); err != nil {
t.Fatal(err)
}
anchor.set(measuredHouse("/var/lib/house/config", 300<<10, time.Now()))
obs := findingsByKind(run())
o, ok := obs[kindEmptyReplacement]
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "laptop" {
t.Fatalf("an empty replacement of a module's data was not urgent: %+v", obs)
}
}
// Data on redundant storage: the array it is on is watched, one condition per array as loud as the most
// precious item on it; an item said to be on redundancy and found on plain storage is said; an item
// whose path is gone is said.
func TestTheArrayUnderDataIsWatched(t *testing.T) {
now := time.Now()
media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}],
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array, no room to copy"},
{"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array, no room to copy"}]}}`
shelf := shelfFor(t, media)
sick := false
on := func(item string, healthy *bool) inventory.DataRecord {
return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable", Owned: false,
Path: "/tank/" + item, Size: bytesOf(40 << 40), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now),
Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: healthy, Said: "pool 'tank' is DEGRADED"}}
}
got := dataFindings([]inventory.DataRecord{on("films", &sick), on("shows", &sick)}, nil, shelf, nil, nil, now)
if len(got) != 1 || got[0].Kind != kindArrayDegraded || got[0].Severity != conditions.Urgent ||
!strings.Contains(got[0].Summary, "media/films, media/shows") {
t.Fatalf("%+v", got)
}
well := true
if got := dataFindings([]inventory.DataRecord{on("films", &well)}, nil, shelf, nil, nil, now); len(got) != 0 {
t.Fatalf("a healthy array raised %+v", got)
}
plain := on("films", nil)
plain.Redundancy = nil
if o := findingsByKind(dataFindings([]inventory.DataRecord{plain}, nil, shelf, nil, nil, now))[kindProtectionMissing]; o.Severity != conditions.Urgent {
t.Fatalf("redundancy said and not found: %+v", o)
}
gone := on("films", &well)
gone.MeasureError, gone.Size = "/tank/films does not exist", bytesOf(0)
if o := findingsByKind(dataFindings([]inventory.DataRecord{gone}, map[string]int64{gone.Key(): 40 << 40}, shelf, nil, nil, now))[kindDataMissing]; o.Severity != conditions.Urgent {
t.Fatalf("a vanished library: %+v", o)
}
}
// What a consumer keeps with its provider as irreplaceable holds the provider's item to that class:
// the photo site's objects make the object store's data an urgent matter.
func TestKeptByHoldsTheProvidersItemToTheConsumersClass(t *testing.T) {
objects := `{"module":"objects","version":"1","provides":[{"name":"s3-bucket","scope":"mesh"}],"grants":{"s3-bucket":"${dir:g}"},
"data":{"own":[{"id":"data","path":"${dir:data}","class":"valuable"}],"consumers":{"s3-bucket":{"class":"valuable","in":"data"}}},
"resources":[{"id":"g","type":"directory","mode":"0700"},{"id":"data","type":"directory","mode":"0700"}]}`
photos := `{"module":"photos","version":"1","requires":["s3-bucket"],"data":{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}}`
shelf := shelfFor(t, objects, photos)
bindings := []inventory.Binding{{Machine: "anchor", Consumer: "photos", Provision: "s3-bucket",
Provider: catalogue.Chosen{Node: "anchor", Module: "objects"}}}
upgraded, keptBy := keptByClasses(bindings, shelf)
if upgraded["anchor/objects/data"] != "irreplaceable" || keptBy["objects/mesh_anchor_photos"] != "irreplaceable" {
t.Fatalf("upgraded %v, kept by %v", upgraded, keptBy)
}
now := time.Now()
r := inventory.DataRecord{Machine: "anchor", Module: "objects", Item: "data", Class: "valuable", Owned: true,
Size: bytesOf(10 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), MeasuredAt: when(now), LastBackup: when(now.Add(-72 * time.Hour))}
if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, upgraded, now))[kindBackupStale]; o.Severity != conditions.Urgent {
t.Fatalf("the photos' store without a backup: %+v", o)
}
if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now))[kindBackupStale]; o.Severity != conditions.Warning {
t.Fatalf("a valuable store without a backup: %+v", o)
}
}
// Items measured from one dataset's counters share its size: a shrink of the dataset is one condition
// naming every item on it, not one per item; and a partial walk's lower bound is never compared.
func TestADatasetShrinksOnceAndAPartialSizeIsNeverCompared(t *testing.T) {
now := time.Now()
media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}],
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array","measure":"dataset"},
{"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array","measure":"dataset"}]}}`
shelf := shelfFor(t, media, houseManifest)
well := true
on := func(item string, size int64) inventory.DataRecord {
return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable",
Size: bytesOf(size), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now),
Precision: "dataset tank/media: its whole size",
Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: &well}}
}
films, shows := on("films", 30<<40), on("shows", 30<<40)
peaks := map[string]int64{films.Key(): 90 << 40, shows.Key(): 90 << 40}
got := dataFindings([]inventory.DataRecord{films, shows}, peaks, shelf, nil, nil, now)
if len(got) != 1 || got[0].Kind != kindDataShrank || got[0].Severity != conditions.Urgent ||
!strings.Contains(got[0].Summary, "media/films, media/shows") {
t.Fatalf("%+v", got)
}
partial := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
Size: bytesOf(1 << 20), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now), LastWrite: when(now),
LastBackup: when(now), Precision: "partial: measured partially"}
if got := dataFindings([]inventory.DataRecord{partial}, map[string]int64{partial.Key(): 1 << 30}, shelf, nil, nil, now); len(got) != 0 {
t.Fatalf("a partial size was compared: %+v", got)
}
}
+8
View File
@@ -102,6 +102,14 @@ var probeRegistry = []probe{
{ID: probeBindingsID, Asserts: "every consumer of a provision that keeps its data is bound where it was last " + {ID: probeBindingsID, Asserts: "every consumer of a provision that keeps its data is bound where it was last " +
"sent, or moves by a pin", From: "issue 273, ADR 0232", Kind: kindBindingMoved, "sent, or moves by a pin", From: "issue 273, ADR 0232", Kind: kindBindingMoved,
Raises: []string{kindBindingKept, kindBindingMoving}, Phase: 2, run: probeBindings}, Raises: []string{kindBindingKept, kindBindingMoving}, Phase: 2, run: probeBindings},
{ID: probeDataID, Asserts: "every item of data a machine declares is measured, is there, holds what it held, is " +
"written where it should be, is backed up within its bound or sits on healthy redundant storage, and is no " +
"empty replacement of a copy kept elsewhere; what a machine no longer declares that is irreplaceable or " +
"valuable is retired, not forgotten", From: "issue 273, ADR 0233",
Kind: kindDataShrank, Raises: []string{kindEmptyReplacement, kindDataHeldTwice, kindDataQuiet, kindBackupStale,
kindDataUnmeasured, kindDataMissing, kindArrayDegraded, kindProtectionMissing, kindCleanupWaiting},
Phase: 2, run: probeData,
Asks: []broker.SeatVerb{{Seat: "node-backup", Verb: "backed-up"}}},
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals", {ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs}, From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
} }
+3
View File
@@ -170,6 +170,9 @@ func run() error {
return retireCommand(ctx, args[1:]) return retireCommand(ctx, args[1:])
case "cleanup": case "cleanup":
return cleanupCommand(ctx, args[1:]) return cleanupCommand(ctx, args[1:])
// The data every machine declares, as the self-check last found it (novox/hq ADR 0233).
case "data":
return dataCommand(ctx, args[1:])
case "version": case "version":
fmt.Println(version) fmt.Println(version)
return nil return nil
+45 -8
View File
@@ -258,10 +258,21 @@ func cleanupCommand(ctx context.Context, args []string) error {
} }
defer open.Close() defer open.Close()
return onTheBus(func(conn *nats.Conn) error { return onTheBus(func(conn *nats.Conn) error {
return deleteOlderThan(ctx, open.inventory, conn, *olderThan, *confirm, f, time.Now()) return deleteOlderThan(ctx, open, conn, *olderThan, *confirm, f, time.Now())
}) })
case *olderThan == 0 && len(rest) == 3 && !*confirm: case *olderThan == 0 && len(rest) == 3 && !*confirm:
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
return onTheBus(func(conn *nats.Conn) error { return onTheBus(func(conn *nats.Conn) error {
// A module's own retired data, when the mesh holds such an item (novox/hq ADR 0233); a
// provider's retired consumer otherwise.
if r, err := open.inventory.DataOf(ctx, rest[0], rest[1], rest[2]); err == nil && r.DeletedAt == nil &&
r.RetiredAt != nil {
return deleteRetiredData(ctx, conn, open, r, f)
}
return deleteRetired(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, rest[2], f) return deleteRetired(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, rest[2], f)
}) })
} }
@@ -285,6 +296,10 @@ type retiredRow struct {
Why string `json:"why,omitempty"` Why string `json:"why,omitempty"`
// Access is "kept" for a consumer of a mark-only provider: retired on record, still reachable. // Access is "kept" for a consumer of a mark-only provider: retired on record, still reachable.
Access string `json:"access,omitempty"` Access string `json:"access,omitempty"`
// Path and Class are a module's own retired data's (Kind own-data, novox/hq ADR 0233): where it is
// kept on its machine, and its class. Consumer is then the item.
Path string `json:"path,omitempty"`
Class string `json:"class,omitempty"`
} }
// retiredListing is every provider's retired consumers, and the providers that could not say. // retiredListing is every provider's retired consumers, and the providers that could not say.
@@ -299,7 +314,15 @@ func gatherRetired(ctx context.Context, inv *inventory.Inventory, conn *nats.Con
if err != nil { if err != nil {
return retiredListing{}, err return retiredListing{}, err
} }
return retiredOf(ctx, conn, instances, now), nil listing := retiredOf(ctx, conn, instances, now)
// And every module's own data retired on its machine (novox/hq ADR 0233).
records, err := inv.Data(ctx)
if err != nil {
return retiredListing{}, err
}
listing.Retired = append(listing.Retired, retiredData(records, now)...)
sort.SliceStable(listing.Retired, func(i, j int) bool { return listing.Retired[i].AgeDays > listing.Retired[j].AgeDays })
return listing, nil
} }
func retiredOf(ctx context.Context, conn *nats.Conn, instances []providerInstance, now time.Time) retiredListing { func retiredOf(ctx context.Context, conn *nats.Conn, instances []providerInstance, now time.Time) retiredListing {
@@ -332,7 +355,7 @@ func printRetired(l retiredListing, asJSON bool) error {
return printJSON(l) return printJSON(l)
} }
if len(l.Retired) == 0 { if len(l.Retired) == 0 {
fmt.Println("no provider holds a retired consumer") fmt.Println("no provider holds a retired consumer, and no machine holds retired data")
} }
for _, r := range l.Retired { for _, r := range l.Retired {
age := "age unknown" age := "age unknown"
@@ -346,6 +369,11 @@ func printRetired(l retiredListing, asJSON bool) error {
if r.Access == "kept" { if r.Access == "kept" {
kind += " [MARK ONLY: access kept until deleted]" kind += " [MARK ONLY: access kept until deleted]"
} }
if r.Kind == retiredDataKind {
fmt.Printf("%s on %s: its own %s, %s, at %s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer,
r.Class, r.Path, age, sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why))
continue
}
fmt.Printf("%s on %s: %s%s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer, kind, age, fmt.Printf("%s on %s: %s%s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer, kind, age,
sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why)) sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why))
} }
@@ -391,16 +419,16 @@ func deleteRetired(ctx context.Context, conn *nats.Conn, p providerInstance, con
// deleteOlderThan lists every consumer retired more than days ago, and deletes them only with confirm. // deleteOlderThan lists every consumer retired more than days ago, and deletes them only with confirm.
// One whose age the provider cannot say is never in it. // One whose age the provider cannot say is never in it.
func deleteOlderThan(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, days int, confirm bool, func deleteOlderThan(ctx context.Context, open *stores, conn *nats.Conn, days int, confirm bool,
f handActFlags, now time.Time) error { f handActFlags, now time.Time) error {
listing, err := gatherRetired(ctx, inv, conn, now) listing, err := gatherRetired(ctx, open.inventory, conn, now)
if err != nil { if err != nil {
return err return err
} }
return deleteFrom(ctx, conn, listing, days, confirm, f) return deleteFrom(ctx, conn, open, listing, days, confirm, f)
} }
func deleteFrom(ctx context.Context, conn *nats.Conn, listing retiredListing, days int, confirm bool, f handActFlags) error { func deleteFrom(ctx context.Context, conn *nats.Conn, open *stores, listing retiredListing, days int, confirm bool, f handActFlags) error {
var due []retiredRow var due []retiredRow
unknown := 0 unknown := 0
for _, r := range listing.Retired { for _, r := range listing.Retired {
@@ -431,7 +459,16 @@ func deleteFrom(ctx context.Context, conn *nats.Conn, listing retiredListing, da
} }
var failed []string var failed []string
for _, r := range due { for _, r := range due {
if err := deleteRetired(ctx, conn, providerInstance{Node: r.Node, Module: r.Module}, r.Consumer, f); err != nil { var err error
if r.Kind == retiredDataKind {
var rec inventory.DataRecord
if rec, err = open.inventory.DataOf(ctx, r.Node, r.Module, r.Consumer); err == nil {
err = deleteRetiredData(ctx, conn, open, rec, f)
}
} else {
err = deleteRetired(ctx, conn, providerInstance{Node: r.Node, Module: r.Module}, r.Consumer, f)
}
if err != nil {
failed = append(failed, err.Error()) failed = append(failed, err.Error())
} }
} }
+2 -2
View File
@@ -380,12 +380,12 @@ func TestNatsCleanupDeletesOnlyTheNamedRetiredConsumer(t *testing.T) {
t.Fatalf("%+v", listing) t.Fatalf("%+v", listing)
} }
fake.deleted = nil fake.deleted = nil
said = printed(t, func() error { return deleteFrom(t.Context(), conn, listing, 30, false, whyFlags(t, "tidy")) }) said = printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, false, whyFlags(t, "tidy")) })
if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") || if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") ||
strings.Contains(said, "young") { strings.Contains(said, "young") {
t.Fatalf("deleted %v; said %s", fake.deleted, said) t.Fatalf("deleted %v; said %s", fake.deleted, said)
} }
printed(t, func() error { return deleteFrom(t.Context(), conn, listing, 30, true, whyFlags(t, "tidy")) }) printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, true, whyFlags(t, "tidy")) })
if !slices.Equal(fake.deleted, []string{"old"}) { if !slices.Equal(fake.deleted, []string{"old"}) {
t.Fatalf("confirmed, deleted %v", fake.deleted) t.Fatalf("confirmed, deleted %v", fake.deleted)
} }
+10 -1
View File
@@ -499,6 +499,15 @@ func (a *verbArguments) commandLine() ([]string, error) {
return argv, nil return argv, nil
} }
return []string{"cleanup", "list", "--json"}, nil return []string{"cleanup", "list", "--json"}, nil
case "data":
argv := []string{"data", "--json"}
if m := str("machine"); m != "" {
argv = append(argv, "--machine", m)
}
if on("retired") {
argv = append(argv, "--retired")
}
return argv, nil
case "doctor": case "doctor":
which := 0 which := 0
argv := []string{"doctor"} argv := []string{"doctor"}
@@ -598,7 +607,7 @@ func (a *verbArguments) commandLine() ([]string, error) {
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well. // jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true, var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true,
"hand-acts": true, "durations": true, "conditions": true, "doctor": true, "retire": true, "cleanup": true} "hand-acts": true, "durations": true, "conditions": true, "doctor": true, "retire": true, "cleanup": true, "data": true}
// repairingCommand names a command line that repairs by hand, and so says why: a push, a plan stopped // repairingCommand names a command line that repairs by hand, and so says why: a push, a plan stopped
// or closed, a consumer re-made (novox/hq to-be 45 §7). Empty for any other. // or closed, a consumer re-made (novox/hq to-be 45 §7). Empty for any other.
@@ -276,6 +276,7 @@ var accountedFlags = map[string]map[string]string{
"conditions": {"json": "set by the verb: the answer is data"}, "conditions": {"json": "set by the verb: the answer is data"},
"retire": {"json": "set by the verb: the answer is data"}, "retire": {"json": "set by the verb: the answer is data"},
"cleanup": {"json": "set by the verb: the answer is data"}, "cleanup": {"json": "set by the verb: the answer is data"},
"data": {"json": "set by the verb: the answer is data"},
"conditions history": {"json": "set by the verb: the answer is data"}, "conditions history": {"json": "set by the verb: the answer is data"},
"conditions show": {"json": "set by the verb: the answer is data"}, "conditions show": {"json": "set by the verb: the answer is data"},
"healers": {"json": "set by the verb: the answer is data"}, "healers": {"json": "set by the verb: the answer is data"},
+5 -1
View File
@@ -133,7 +133,11 @@ type SeatVerb struct{ Seat, Verb string }
// to the probe registry is the reason they cannot drift** — a probe that calls a verb its grant does // to the probe registry is the reason they cannot drift** — a probe that calls a verb its grant does
// not name is refused by the bus on every run (found live on 2026-10-06: D8 timed out on each // not name is refused by the bus on every run (found live on 2026-10-06: D8 timed out on each
// machine, refused). Asked of any machine (`.*`), read-only verbs, nothing else of the seat. // machine, refused). Asked of any machine (`.*`), read-only verbs, nothing else of the seat.
var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}} //
// D13 reads every machine's backup holder: what it measured of the data declared there (novox/hq ADR
// 0233).
var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"},
{Seat: "node-backup", Verb: "backed-up"}}
// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the // perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
// holder's machine (novox/hq ADR 0219): `paused.<node>`, the build agent saying whether it takes work. // holder's machine (novox/hq ADR 0219): `paused.<node>`, the build agent saying whether it takes work.
+1 -1
View File
@@ -24,7 +24,7 @@ accounts {
jetstream: enabled jetstream: enabled
users = [ users = [
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] } publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] } subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
allow_responses: { max: 1, ttl: "1m" } allow_responses: { max: 1, ttl: "1m" }
} } } }
+8 -33
View File
@@ -5,35 +5,10 @@ import (
"testing" "testing"
) )
// A store provider says how its data is backed up (novox/hq ADR 0214); a provider of something that // A backup contribution written by hand still names its module's own directories; one it does not
// holds nothing does not have to. // declare is refused where it is written rather than reaching the holder as the literal text. (The
func TestAStoreProviderWithoutABackupIsRefusedByTheCheck(t *testing.T) { // catalogue check refuses a hand-written backup line at all since ADR 0233; parsing still reads one,
bare, err := ParseManifest([]byte(`{"module":"pg","version":"1", // because a module on the shelf was written before.)
"provides":[{"name":"postgres-database","scope":"mesh"}]}`))
if err != nil {
t.Fatalf("parsing refused it, and the rule is the check's: %v", err)
}
if problems := CheckBackup(bare); len(problems) != 1 || !strings.Contains(problems[0], "node-backup") {
t.Fatalf("a store with no backup passed the check: %v", problems)
}
backed, err := ParseManifest([]byte(`{"module":"pg","version":"1",
"provides":[{"name":"postgres-database","scope":"mesh"}],
"resources":[{"id":"dumps","type":"directory","mode":"0700"}],
"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:dumps}"}]}`))
if err != nil {
t.Fatal(err)
}
if problems := CheckBackup(backed); len(problems) != 0 {
t.Fatalf("a store that contributes a backup was refused: %v", problems)
}
route, _ := ParseManifest([]byte(`{"module":"r","version":"1","provides":[{"name":"route","scope":"mesh"}]}`))
if problems := CheckBackup(route); len(problems) != 0 {
t.Fatalf("a route was asked for a backup: %v", problems)
}
}
// A backup contribution names its module's own directories; one it does not declare is refused
// where it is written rather than reaching the holder as the literal text.
func TestABackupNamingAnUndeclaredDirectoryIsRefused(t *testing.T) { func TestABackupNamingAnUndeclaredDirectoryIsRefused(t *testing.T) {
_, err := ParseManifest([]byte(`{"module":"pg","version":"1", _, err := ParseManifest([]byte(`{"module":"pg","version":"1",
"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:nowhere}"}]}`)) "contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:nowhere}"}]}`))
@@ -42,9 +17,9 @@ func TestABackupNamingAnUndeclaredDirectoryIsRefused(t *testing.T) {
} }
} }
// The holder receives every module's backup lines with each module's own directories filled, each // A module on the shelf from before ADR 0233, which writes its backup lines by hand and declares no
// module's under a comment naming it — and a kind written in a shell's grammar is left untouched. // data, is still backed up: its lines are placed as written, each module's under a comment naming it.
func TestBackupContributionsArePlacedWithTheirModulesDirectories(t *testing.T) { func TestHandWrittenBackupLinesOfAnOlderModuleArePlaced(t *testing.T) {
pg, err := ParseManifest([]byte(`{"module":"pg","version":"1", pg, err := ParseManifest([]byte(`{"module":"pg","version":"1",
"resources":[{"id":"dumps","type":"directory","path":"/srv/pg/dumps","mode":"0700"}], "resources":[{"id":"dumps","type":"directory","path":"/srv/pg/dumps","mode":"0700"}],
"contributions":[{"seat":"node-backup","kind":"backup","content":"run pg-dump-all\npath ${dir:dumps}"}]}`)) "contributions":[{"seat":"node-backup","kind":"backup","content":"run pg-dump-all\npath ${dir:dumps}"}]}`))
@@ -57,7 +32,7 @@ func TestBackupContributionsArePlacedWithTheirModulesDirectories(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
placed, err := seatContributions([]Manifest{pg, mail}, BackupSeat, "backup", Rendering{}) placed, err := seatContributions([]Manifest{pg, mail}, BackupSeat, "backup", Rendering{}, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
+15 -14
View File
@@ -87,15 +87,18 @@ func TestNoCatalogueManifestNamesAnInstallation(t *testing.T) {
} }
} }
// TestEveryCatalogueStoreSaysHowItIsBackedUp is ADR 0214's check over the real catalogue: a module // TestEveryCatalogueModuleDeclaresItsData is ADR 0233's check over the real catalogue (it replaced ADR
// providing a store contributes a backup to node-backup, so a store added is a store backed up. // 0214's store list): every provider that grants says what it keeps for its consumers, nothing writes a
func TestEveryCatalogueStoreSaysHowItIsBackedUp(t *testing.T) { // backup line by hand, every directory a container writes is declared, and every irreplaceable item is
// backed up — so a store added is a store backed up, without a list of stores to keep in step.
func TestEveryCatalogueModuleDeclaresItsData(t *testing.T) {
root := catalogueRoot(t) root := catalogueRoot(t)
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
if err != nil || len(found) == 0 { if err != nil || len(found) == 0 {
t.Fatalf("no manifests under %s: %v", root, err) t.Fatalf("no manifests under %s: %v", root, err)
} }
stores := 0 shelf := Shelf{}
granting := 0
for _, p := range found { for _, p := range found {
raw, err := os.ReadFile(p) raw, err := os.ReadFile(p)
if err != nil { if err != nil {
@@ -105,18 +108,16 @@ func TestEveryCatalogueStoreSaysHowItIsBackedUp(t *testing.T) {
if err != nil { if err != nil {
continue // TestEveryCatalogueManifestParses says why continue // TestEveryCatalogueManifestParses says why
} }
for _, o := range m.Provides { if len(m.Grants) > 0 {
if storeProvisions[o.Name] { granting++
stores++
break
}
}
for _, problem := range CheckBackup(m) {
t.Error(problem)
} }
shelf[m.Module] = m
} }
if stores == 0 { for _, problem := range DataProblems(shelf) {
t.Fatal("no module in the catalogue provides a store, so this proved nothing") t.Error(problem)
}
if granting == 0 {
t.Fatal("no module in the catalogue grants a provision, so this proved nothing")
} }
} }
+791
View File
@@ -0,0 +1,791 @@
package catalogue
import (
"bytes"
"encoding/json"
"fmt"
"regexp"
"sort"
"strconv"
"strings"
"time"
)
// A module declares the data it holds, and the mesh protects and watches it from that declaration
// (novox/hq ADR 0233).
//
// **One section, `data`, for every kind of data a module keeps:** its own, by directory — a store's
// files, a mail spool, an application's uploads — or by an operator's path it was given; what it keeps
// for its consumers, by the provision they reach it through; and what of its own lives with a
// provider, by the provision it requires. Each entry has a class — how precious it is — and, for its
// own data, how it is protected; everything the mesh does is derived from those, never written per
// module:
//
// - a binding to a consumer's data does not move (ADR 0232) where the provision's class keeps data;
// - the backup holder's lines are composed from it — a module no longer writes them by hand;
// - what an unassignment leaves behind of an irreplaceable or valuable item is retired, listed by
// `cleanup list` and deleted only by `cleanup delete` (ADR 0230);
// - the self-check measures every item and says when one shrinks, disappears, stops being written,
// goes without its backup, sits on a degraded array, or is replaced by an empty copy of itself —
// urgent for what is irreplaceable, a warning for what is valuable.
// The classes: how precious the data is. A fixed vocabulary, ranked by the operator (2026-10-06): a
// class is what the protections are derived from, so a new one is a decision, not a manifest's choice.
const (
// ClassIrreplaceable is what must never be lost: the operator names it (the media library, the
// photo sites' storage). Protected by a backup or by a declared redundancy — one is required —
// retired rather than removed, and every alert about it is urgent.
ClassIrreplaceable = "irreplaceable"
// ClassValuable is anybody's work that would be painful to lose: in the nightly backup by default,
// retired rather than removed, and every alert about it a warning.
ClassValuable = "valuable"
// ClassRebuildable can be made again from something kept elsewhere — a clone, an index, a
// download, a night's dump — at a cost in time, not in data. In the nightly backup by default;
// forgotten when its module goes, and never alerted on.
ClassRebuildable = "rebuildable"
// ClassCache may be emptied at any moment with nothing lost but speed: never backed up, never
// measured, never alerted on.
ClassCache = "cache"
// ClassNone is a provision that keeps nothing of its consumers' (consumers only): a resolver, a
// certificate authority, an artifact store.
ClassNone = "none"
)
// classRank orders the classes by how precious they are, so the stricter of two is chosen.
var classRank = map[string]int{ClassNone: 0, ClassCache: 1, ClassRebuildable: 2, ClassValuable: 3, ClassIrreplaceable: 4}
// StricterClass is the more precious of two classes.
func StricterClass(a, b string) string {
if classRank[b] > classRank[a] {
return b
}
return a
}
// Retires is whether a class's data is retired, not forgotten, when its machine no longer declares it.
func Retires(class string) bool { return class == ClassIrreplaceable || class == ClassValuable }
// Watched is whether a class's data raises conditions: irreplaceable and valuable.
func Watched(class string) bool { return Retires(class) }
// DefaultBackupWithin is how old the last good backup of an item may be before the self-check says so
// (novox/hq ADR 0214: a machine with data and no good backup in 48 hours).
const DefaultBackupWithin = 48 * time.Hour
// Data is a manifest's `data` section.
type Data struct {
// Own is the data this module keeps itself.
Own []DataItem `json:"own,omitempty"`
// Consumers is what it keeps for its consumers, per provision it grants.
Consumers map[string]ConsumerData `json:"consumers,omitempty"`
// KeptBy is what of its own lives with the provider of a provision it requires — its rows, its
// objects — and how precious that is. The provider's protections follow the stricter of its own
// class for its consumers and this.
KeptBy map[string]KeptData `json:"kept-by,omitempty"`
}
// DataItem is one piece of a module's own data.
type DataItem struct {
// ID names it within the module: what `data`, `cleanup` and a condition call it.
ID string `json:"id"`
// Path is one of the module's directories, `${dir:<id>}`, or an operator's path it was given,
// `${access:<id>}`, or a path beneath either. Never a machine path (novox/hq ADR 0112).
Path string `json:"path"`
Class string `json:"class"`
// Backup is how it is copied: "copy" (the holder reads it as it stands), "none", or a dump — a
// command that writes a consistent copy into another item, which is copied. Unsaid, anything but a
// cache is copied, unless it says it is protected by redundancy instead.
Backup *DataBackup `json:"backup,omitempty"`
// Redundancy says it is protected by the redundancy of the storage it lives on rather than by a
// copy, and why that is enough: the media library on an array there is no room to copy. The
// backup holder then watches that array, and a degraded one is said.
Redundancy string `json:"redundancy,omitempty"`
// Within is how old its last good backup may be; unsaid, DefaultBackupWithin.
Within string `json:"within,omitempty"`
// Measure is how the backup holder measures it: `walk` (the default — every file, at most daily and
// bounded, for small items), `dataset` (a ZFS dataset's own counters, hourly, nothing walked) or
// `shallow` (its top-level entries only, no size). A large item never says walk.
Measure string `json:"measure,omitempty"`
// Active is how long it may go unwritten before that is a fault — for data something is
// expected to write all the time. Unsaid, a quiet item is not a fault.
Active string `json:"active,omitempty"`
// Why is a line for the reviewer: why this class.
Why string `json:"why,omitempty"`
}
// ConsumerData is what a provider keeps for the consumers of one provision.
type ConsumerData struct {
Class string `json:"class"`
// In is where it lives: one of the module's own items (whose protection covers it), or a
// provision this module requires (whose provider keeps it, as its consumer). Unsaid only for none
// and cache.
In string `json:"in,omitempty"`
Why string `json:"why,omitempty"`
}
// KeptData is how precious what a module keeps with a provider is.
type KeptData struct {
Class string `json:"class"`
Why string `json:"why,omitempty"`
}
// DataBackup is how an item is copied.
type DataBackup struct {
Copy bool
None bool
// Dump is the command writing a consistent copy into the item Into.
Dump string
Into string
}
// UnmarshalJSON reads "copy", "none" or {"dump": "...", "into": "<item>"}.
func (b *DataBackup) UnmarshalJSON(raw []byte) error {
var word string
if err := json.Unmarshal(raw, &word); err == nil {
switch word {
case "copy":
*b = DataBackup{Copy: true}
case "none":
*b = DataBackup{None: true}
default:
return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}, not %q", word)
}
return nil
}
var full struct {
Dump string `json:"dump"`
Into string `json:"into"`
}
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&full); err != nil {
return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}: %w", err)
}
*b = DataBackup{Dump: full.Dump, Into: full.Into}
return nil
}
// MarshalJSON writes it back in the form it was written.
func (b DataBackup) MarshalJSON() ([]byte, error) {
switch {
case b.Copy:
return json.Marshal("copy")
case b.None:
return json.Marshal("none")
}
return json.Marshal(struct {
Dump string `json:"dump"`
Into string `json:"into"`
}{b.Dump, b.Into})
}
// IsDump is whether the item is copied by a dump.
func (b *DataBackup) IsDump() bool { return b != nil && b.Dump != "" }
// BackedUp is whether the holder keeps restore points of this item: anything but a cache by default —
// the nightly backup is the standard plan — unless it says "none", or says it is protected by
// redundancy and says nothing of a backup.
func (it DataItem) BackedUp() bool {
switch {
case it.Backup == nil:
return it.Class != ClassCache && it.Redundancy == ""
case it.Backup.None:
return false
}
return true
}
// Protection is how the item is protected, in one word: "backup", "redundancy", both joined by "+",
// or "none".
func (it DataItem) Protection() string {
var by []string
if it.BackedUp() {
by = append(by, "backup")
}
if it.Redundancy != "" {
by = append(by, "redundancy")
}
if len(by) == 0 {
return "none"
}
return strings.Join(by, "+")
}
// The ways an item is measured.
const (
MeasureWalk = "walk"
MeasureDataset = "dataset"
MeasureShallow = "shallow"
)
// MeasuredBy is how the item is measured, with the default applied.
func (it DataItem) MeasuredBy() string {
if it.Measure == "" {
return MeasureWalk
}
return it.Measure
}
// OwnedByModule is whether the item is in one of the module's own directories — the mesh's to retire —
// rather than an operator's path it was given, which the mesh never retires and never deletes.
func (it DataItem) OwnedByModule() bool { return strings.HasPrefix(it.Path, "${dir:") }
// BackupWithin is the item's bound on its last good backup.
func (it DataItem) BackupWithin() time.Duration {
if d, err := ParseDataDuration(it.Within); err == nil && d > 0 {
return d
}
return DefaultBackupWithin
}
// ActiveWithin is how long the item may go unwritten; zero when quiet is not a fault.
func (it DataItem) ActiveWithin() time.Duration {
d, _ := ParseDataDuration(it.Active)
return d
}
// ParseDataDuration reads "48h", "90m" or "7d"; empty is zero.
func ParseDataDuration(s string) (time.Duration, error) {
s = strings.TrimSpace(s)
if s == "" {
return 0, nil
}
if days, ok := strings.CutSuffix(s, "d"); ok {
n, err := strconv.Atoi(days)
if err != nil || n <= 0 {
return 0, fmt.Errorf("%q is not a number of days", s)
}
return time.Duration(n) * 24 * time.Hour, nil
}
d, err := time.ParseDuration(s)
if err != nil || d <= 0 {
return 0, fmt.Errorf("%q is not a duration (48h, 90m, 7d)", s)
}
return d, nil
}
// DataItems is the module's own data, in the order declared.
func (m Manifest) DataItems() []DataItem {
if m.Data == nil {
return nil
}
return m.Data.Own
}
// DataItem is one own item by id.
func (m Manifest) DataItem(id string) (DataItem, bool) {
for _, it := range m.DataItems() {
if it.ID == id {
return it, true
}
}
return DataItem{}, false
}
// ConsumerDataOf is what this module says it keeps for a provision's consumers, and whether it says.
func (m Manifest) ConsumerDataOf(provision string) (ConsumerData, bool) {
if m.Data == nil {
return ConsumerData{}, false
}
c, ok := m.Data.Consumers[provision]
return c, ok
}
// KeptByOf is how precious what this module keeps with a provision's provider is, and whether it says.
func (m Manifest) KeptByOf(provision string) (KeptData, bool) {
if m.Data == nil {
return KeptData{}, false
}
k, ok := m.Data.KeptBy[provision]
return k, ok
}
// keepsByClass is whether a class keeps something a binding must not move away from.
func keepsByClass(class string) bool {
return class == ClassIrreplaceable || class == ClassValuable || class == ClassRebuildable
}
// dataID is what an item's id may be: it is a token in a condition's key and a word on a line.
var dataID = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
// dataPathRef is an item's path: one of the module's directories or accesses, and optionally a path
// beneath it.
var dataPathRef = regexp.MustCompile(`^\$\{(dir|access):([a-z0-9][a-z0-9-]*)\}(/[^\s$]*)?$`)
// dataProblems is what is wrong with the `data` section itself, from the manifest alone: judged at
// registration as every other per-manifest problem is. Whether a module declares what it should is
// the catalogue check's (DataProblems), because a module already running was written before it.
func (m Manifest) dataProblems() []string {
if m.Data == nil {
return nil
}
var problems []string
say := func(format string, args ...any) {
problems = append(problems, fmt.Sprintf("%s's data: ", m.Module)+fmt.Sprintf(format, args...))
}
dirs := map[string]bool{}
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) == "directory" {
dirs[fmt.Sprint(r["id"])] = true
}
}
accesses := map[string]bool{}
for _, a := range m.Accesses {
if a.ID != "" {
accesses[a.ID] = true
}
}
ids := map[string]DataItem{}
paths := map[string]string{}
for i, it := range m.Data.Own {
label := it.ID
if label == "" {
label = fmt.Sprintf("item %d", i+1)
}
if !dataID.MatchString(it.ID) {
say("%s has no usable id: lower-case letters, digits and dashes", label)
} else if _, twice := ids[it.ID]; twice {
say("%s is declared twice", it.ID)
}
ids[it.ID] = it
ref := dataPathRef.FindStringSubmatch(it.Path)
switch {
case ref == nil:
say("%s's path %q is not one of the module's directories or accesses: ${dir:<id>} or ${access:<id>}, "+
"or a path beneath one (a definition names no machine path, novox/hq ADR 0112)", label, it.Path)
case ref[1] == "dir" && !dirs[ref[2]]:
say("%s's path names ${dir:%s}, and %s declares no directory %q", label, ref[2], m.Module, ref[2])
case ref[1] == "access" && !accesses[ref[2]]:
say("%s's path names ${access:%s}, and %s declares no access %q", label, ref[2], m.Module, ref[2])
case strings.Contains(it.Path, ".."):
say("%s's path %q climbs out of its directory", label, it.Path)
}
if other, twice := paths[it.Path]; twice && it.Path != "" {
say("%s and %s name the same path %s", other, label, it.Path)
}
paths[it.Path] = label
switch it.Class {
case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache:
case ClassNone:
say("%s is class none, which only a provision keeping nothing of its consumers' is; own data "+
"that is disposable is cache", label)
default:
say("%s's class %q is not one the mesh protects by: irreplaceable, valuable, rebuildable or cache",
label, it.Class)
}
if it.Class == ClassIrreplaceable && !it.BackedUp() && strings.TrimSpace(it.Redundancy) == "" {
say("%s is irreplaceable and is neither backed up nor said to be protected by redundancy; the only "+
"copy of something is protected one way or the other (novox/hq ADR 0233) — copy it, dump it into "+
"another item, or say `redundancy` with why that is enough", label)
}
if it.Class == ClassCache && it.Backup != nil && !it.Backup.None {
say("%s is a cache and asks to be backed up; a cache is disposable, or it is not a cache", label)
}
if it.Class == ClassCache && it.Redundancy != "" {
say("%s is a cache and says it is protected by redundancy; a cache is not protected", label)
}
switch it.Measure {
case "", MeasureWalk, MeasureDataset, MeasureShallow:
default:
say("%s's measure %q is walk, dataset or shallow", label, it.Measure)
}
if _, err := ParseDataDuration(it.Within); err != nil {
say("%s's within: %v", label, err)
}
if _, err := ParseDataDuration(it.Active); err != nil {
say("%s's active: %v", label, err)
}
if it.Within != "" && !it.BackedUp() {
say("%s states within, and is not backed up", label)
}
if it.Active != "" && !Watched(it.Class) {
say("%s is %s and expects writes; only irreplaceable and valuable data is watched", label, it.Class)
}
}
// Dumps go into an item of the same module, declared, and not into themselves.
for _, it := range m.Data.Own {
if it.Backup == nil || it.Backup.Copy || it.Backup.None {
continue
}
switch into, ok := ids[it.Backup.Into]; {
case strings.TrimSpace(it.Backup.Dump) == "":
say("%s's backup names no dump command", it.ID)
case it.Backup.Into == "":
say("%s's dump says no item it writes into", it.ID)
case !ok:
say("%s's dump writes into %q, which is not one of the module's data items", it.ID, it.Backup.Into)
case into.ID == it.ID:
say("%s's dump writes into itself; a dump is copied from where it lands", it.ID)
case into.Class == ClassCache:
say("%s's dump writes into %s, a cache; what is copied must not be disposable", it.ID, into.ID)
}
if it.Backup.Dump != "" {
declared := map[string]string{}
for d := range dirs {
declared[d] = ""
}
if _, err := dirFill(it.Backup.Dump, declared, m.Module); err != nil {
say("%s's dump: %v", it.ID, err)
}
}
}
provided := map[string]bool{}
for _, o := range m.Provides {
provided[o.Name] = true
}
wants := map[string]bool{}
for _, w := range m.Wants() {
wants[w] = true
}
for _, provision := range sortedKeys(m.Data.Consumers) {
c := m.Data.Consumers[provision]
if !provided[provision] {
say("says what it keeps for the consumers of %q, which it does not provide", provision)
}
switch c.Class {
case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache, ClassNone:
default:
say("its consumers' %s is class %q; one of irreplaceable, valuable, rebuildable, cache or none", provision, c.Class)
}
switch {
case c.Class == ClassNone && c.In != "":
say("its consumers' %s keeps nothing and says where it is kept (%s)", provision, c.In)
case keepsByClass(c.Class) && c.In == "":
say("its consumers' %s is %s and says nowhere it lives: `in` names one of the module's items, or a "+
"provision it requires", provision, c.Class)
case c.In != "":
if own, ok := ids[c.In]; ok {
if c.Class == ClassIrreplaceable && !own.BackedUp() && own.Redundancy == "" {
say("its consumers' %s is irreplaceable and lives in %s, which is not protected", provision, c.In)
}
if classRank[own.Class] < classRank[c.Class] {
say("its consumers' %s is %s and lives in %s, which is only %s", provision, c.Class, c.In, own.Class)
}
} else if !wants[c.In] {
say("its consumers' %s lives in %q, which is neither one of its data items nor a provision it "+
"requires", provision, c.In)
}
}
}
for _, provision := range sortedKeys(m.Data.KeptBy) {
k := m.Data.KeptBy[provision]
if !wants[provision] {
say("says it keeps data with the provider of %q, which it does not require", provision)
}
switch k.Class {
case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache:
default:
say("what it keeps with %s is class %q; one of irreplaceable, valuable, rebuildable or cache", provision, k.Class)
}
}
return problems
}
// KeepsConsumerData is whether this module, providing a provision, keeps what each consumer writes
// there (novox/hq ADR 0232, ADR 0233): whether a consumer bound to it is bound to its data.
//
// **What the data section says, it gets**: irreplaceable, valuable or rebuildable keeps; cache and none
// do not — a cache lost in a move costs speed, not data. Before the section, an offer said it with
// `keeps-consumer-data`, which is still read; unsaid in both, a provider that grants each consumer a
// credential of its own keeps that consumer's data (ADR 0232 §1). The catalogue check refuses the
// unsaid case for a provider that grants (DataProblems), so the inference is what an older definition
// on the shelf gets, never a new one.
func (m Manifest) KeepsConsumerData(provision string) bool {
if c, ok := m.ConsumerDataOf(provision); ok {
return keepsByClass(c.Class)
}
for _, o := range m.Provides {
if o.Name == provision && o.KeepsConsumerData != nil {
return *o.KeepsConsumerData
}
}
_, grants := m.Grants[provision]
return grants
}
// NeedsBackupHolder is whether a module's data needs the machine's backup holder to be there: it keeps
// something irreplaceable — backed up by the holder, or protected by an array the holder watches. A
// module keeping only valuable or rebuildable data is backed up where a holder is, and refused nowhere
// for want of one: the standard plan, not a requirement.
func (m Manifest) NeedsBackupHolder() bool {
for _, it := range m.DataItems() {
if it.Class == ClassIrreplaceable {
return true
}
}
return false
}
// --- derived: the backup holder's lines ---------------------------------------------------------
// The node-backup seat's kinds (novox/hq to-be 43, ADR 0233): `backup` is what to run and which paths
// to keep, `data` every item with its class and protection, for the holder to measure and watch.
const (
BackupKindBackup = "backup"
BackupKindData = "data"
)
// derivedContributions is what a module's data section gives the backup holder: its backup lines and
// its items. Every item is listed, so a valuable one on a machine is measured whether or not it is
// copied; a cache is listed and not measured.
func (m Manifest) derivedContributions() []SeatContribution {
items := m.DataItems()
if len(items) == 0 {
return nil
}
var lines []string
kept := map[string]bool{}
keep := func(path string) {
if !kept[path] {
kept[path] = true
lines = append(lines, "path "+path)
}
}
for _, it := range items {
if !it.BackedUp() {
continue
}
if it.Backup.IsDump() {
lines = append(lines, "run "+strings.TrimSpace(it.Backup.Dump))
if into, ok := m.DataItem(it.Backup.Into); ok {
keep(into.Path)
}
continue
}
keep(it.Path)
}
var described []string
for _, it := range items {
covered := "-"
switch {
case it.Backup.IsDump():
if into, ok := m.DataItem(it.Backup.Into); ok {
covered = into.Path
}
case it.BackedUp():
covered = it.Path
}
described = append(described, fmt.Sprintf("item %s %s %s %s %s %s", it.ID, it.Class, it.Path, covered,
it.Protection(), it.MeasuredBy()))
}
var out []SeatContribution
if len(lines) > 0 {
out = append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindBackup, Content: strings.Join(lines, "\n") + "\n"})
}
return append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindData, Content: strings.Join(described, "\n") + "\n"})
}
// allContributions is every contribution a module makes to a seat's holder: what it wrote, and what
// its data section derives. **One list**: a module that declares its data has its backup lines
// composed from it, and a backup line it still writes by hand is not placed — the catalogue check
// refuses it — so the holder never copies two lists that disagree.
func (m Manifest) allContributions() []SeatContribution {
if m.Data == nil {
return m.Contributions
}
derived := m.derivedContributions()
out := make([]SeatContribution, 0, len(m.Contributions)+len(derived))
for _, c := range m.Contributions {
if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat {
continue
}
out = append(out, c)
}
return append(out, derived...)
}
// --- the catalogue check ------------------------------------------------------------------------
// DataProblems is what the catalogue check refuses about the data a module declares (novox/hq ADR
// 0233), judged over the manifests given together:
//
// - a provider that grants a provision says what it keeps for that provision's consumers;
// - nobody says it on the offer any more (`keeps-consumer-data`): the data section is the one place;
// - nobody writes a backup line by hand: it is derived from the data section;
// - a directory a container writes, mounted whole, is a data item of some class;
// - consumer data said to live in a required provision lives where that provision's provider keeps
// its consumers' data, as preciously, when the provider is given;
// - data a consumer keeps with a provider as irreplaceable is protected there, when the provider is
// given.
//
// **The check's, not registration's**, as ADR 0214's backup rule was: a module already on the shelf
// was written before the rule, and refusing it there would refuse the very providers whose data the
// rule protects. Each module meets it where it is written.
func DataProblems(shelf Shelf) []string {
var problems []string
for _, name := range shelfOrder(shelf) {
m := shelf[name]
for _, provision := range sortedKeys(m.Grants) {
if _, said := m.ConsumerDataOf(provision); !said {
problems = append(problems, fmt.Sprintf(
"%s grants %s and does not say what it keeps for its consumers: data.consumers.%s with a "+
"class — irreplaceable, valuable, rebuildable, cache, or none (novox/hq ADR 0233)", name, provision, provision))
}
}
for _, o := range m.Provides {
if o.KeepsConsumerData != nil {
problems = append(problems, fmt.Sprintf(
"%s says keeps-consumer-data on its offer of %s; it is said once, in data.consumers.%s, with a "+
"class (novox/hq ADR 0233)", name, o.Name, o.Name))
}
}
for i, c := range m.Contributions {
if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat {
problems = append(problems, fmt.Sprintf(
"%s's contribution %d is a backup line written by hand; backups are derived from the data "+
"section — declare the data, with its class and how it is protected (novox/hq ADR 0233)", name, i+1))
}
}
for _, dir := range writtenDirectories(m) {
if !coversDirectory(m, dir) {
problems = append(problems, fmt.Sprintf(
"%s mounts its directory %q into a container to be written, and declares no data in it: "+
"data.own with a class — cache if it is disposable (novox/hq ADR 0233)", name, dir))
}
}
if m.Data == nil {
continue
}
for _, provision := range sortedKeys(m.Data.Consumers) {
c := m.Data.Consumers[provision]
if c.In == "" {
continue
}
if _, own := m.DataItem(c.In); own {
continue
}
for _, pname := range shelfOrder(shelf) {
p := shelf[pname]
pc, said := p.ConsumerDataOf(c.In)
if !said || !providesName(p, c.In) {
continue
}
if classRank[pc.Class] < classRank[c.Class] {
problems = append(problems, fmt.Sprintf(
"%s keeps its consumers' %s, %s, in %s — and %s keeps its consumers' %s as %s, "+
"so it is not protected as %s", name, provision, c.Class, c.In, pname, c.In, pc.Class, c.Class))
}
}
}
for _, provision := range sortedKeys(m.Data.KeptBy) {
k := m.Data.KeptBy[provision]
if k.Class != ClassIrreplaceable {
continue
}
for _, pname := range shelfOrder(shelf) {
p := shelf[pname]
pc, said := p.ConsumerDataOf(provision)
if !said || !providesName(p, provision) {
continue
}
if !keepsByClass(pc.Class) {
problems = append(problems, fmt.Sprintf(
"%s keeps irreplaceable data with %s, and %s keeps its consumers' %s as %s — nothing of it is "+
"protected there", name, provision, pname, provision, pc.Class))
continue
}
if in, own := p.DataItem(pc.In); own && !in.BackedUp() && in.Redundancy == "" {
problems = append(problems, fmt.Sprintf(
"%s keeps irreplaceable data with %s, and %s keeps it in %s, which is neither backed up nor "+
"on declared redundancy", name, provision, pname, pc.In))
}
}
}
}
return problems
}
func providesName(m Manifest, provision string) bool {
for _, o := range m.Provides {
if o.Name == provision {
return true
}
}
return false
}
// writtenDirectories are the module's directories a container mounts whole and may write: a volume
// `${dir:<id>}:<target>` without `:ro`, or a directory stated by an absolute path mounted the same
// way. A file beneath a directory, or a read-only mount, is configuration the mesh wrote.
func writtenDirectories(m Manifest) []string {
absolute := map[string]string{}
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) != "directory" {
continue
}
if p, ok := r["path"].(string); ok && strings.HasPrefix(p, "/") {
absolute[strings.TrimRight(p, "/")] = fmt.Sprint(r["id"])
}
}
seen := map[string]bool{}
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) != "container" {
continue
}
vols, _ := r["volumes"].([]any)
for _, v := range vols {
s, _ := v.(string)
mount := volumeMount.FindStringSubmatch(s)
if mount == nil || volumeReadOnly(mount[3]) {
continue
}
src := strings.TrimRight(mount[1], "/")
if ref := dirPlain.FindStringSubmatch(src); ref != nil {
seen[ref[1]] = true
} else if id, ok := absolute[src]; ok {
seen[id] = true
}
}
}
out := make([]string, 0, len(seen))
for id := range seen {
out = append(out, id)
}
sort.Strings(out)
return out
}
// volumeMount is a container's volume, `<source>:<target>[:<options>]`, its source possibly a
// `${dir:<id>}` — whose own colon is not the separator.
var volumeMount = regexp.MustCompile(`^(\$\{(?:dir|access):[a-z0-9][a-z0-9-]*\}[^:]*|[^:]+):([^:]+)(?::(.*))?$`)
// volumeReadOnly is whether a volume's options mount it read-only.
func volumeReadOnly(options string) bool {
for _, o := range strings.Split(options, ",") {
if strings.TrimSpace(o) == "ro" {
return true
}
}
return false
}
// dirPlain is a whole directory, `${dir:<id>}` and nothing after it.
var dirPlain = regexp.MustCompile(`^\$\{dir:([a-z0-9][a-z0-9-]*)\}$`)
// coversDirectory is whether a data item names the directory, a path within it, or a directory it
// is placed beneath.
func coversDirectory(m Manifest, dir string) bool {
parents := map[string]string{}
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) != "directory" {
continue
}
if p, ok := r["path"].(string); ok {
if ref := dirRef.FindStringSubmatch(p); ref != nil && strings.HasPrefix(p, "${dir:") {
parents[fmt.Sprint(r["id"])] = ref[1]
}
}
}
for _, it := range m.DataItems() {
ref := dataPathRef.FindStringSubmatch(it.Path)
if ref == nil || ref[1] != "dir" {
continue
}
for d, hops := dir, 0; d != "" && hops < 4; d, hops = parents[d], hops+1 {
if ref[2] == d {
return true
}
}
}
return false
}
+227
View File
@@ -0,0 +1,227 @@
package catalogue
import (
"slices"
"strings"
"testing"
)
// A store as a module now declares it (novox/hq ADR 0233): its own data by directory with a class and a
// dump, and what it keeps for its consumers.
const declaredStore = `{"module":"pg","version":"1",
"provides":[{"name":"postgres-database","scope":"mesh"}],
"grants":{"postgres-database":"${dir:grants}"},
"data":{
"own":[
{"id":"store","path":"${dir:store}","class":"irreplaceable","backup":{"dump":"dump-all > ${dir:dumps}/all","into":"dumps"},"active":"1d"},
{"id":"dumps","path":"${dir:dumps}","class":"rebuildable","backup":"none"}],
"consumers":{"postgres-database":{"class":"irreplaceable","in":"store"}}},
"resources":[
{"id":"grants","type":"directory","mode":"0700"},
{"id":"store","type":"directory","path":"/srv/store","mode":"0700"},
{"id":"dumps","type":"directory","path":"${dir:store}/dumps","mode":"0700"},
{"id":"server","type":"container","image":"x@sha256:aa","volumes":["${dir:store}:/var/lib/postgresql/data"]}]}`
func mustParse(t *testing.T, raw string) Manifest {
t.Helper()
m, err := ParseManifest([]byte(raw))
if err != nil {
t.Fatalf("refused: %v", err)
}
return m
}
func TestADeclaredStoreParsesAndPassesTheCheck(t *testing.T) {
m := mustParse(t, declaredStore)
if problems := DataProblems(Shelf{"pg": m}); len(problems) > 0 {
t.Fatalf("a store declaring its data was refused: %v", problems)
}
if !m.KeepsConsumerData("postgres-database") {
t.Fatal("an irreplaceable consumer class does not keep the consumer's data")
}
if it, _ := m.DataItem("store"); !it.BackedUp() || it.BackupWithin() != DefaultBackupWithin || it.ActiveWithin().Hours() != 24 {
t.Fatalf("the store item reads %+v", it)
}
if it, _ := m.DataItem("dumps"); it.BackedUp() {
t.Fatal("a rebuildable item that says none is backed up")
}
}
// Every rule of the section itself, refused at parse in the words of the module.
func TestTheDataSectionIsRefusedWhereItIsWrong(t *testing.T) {
for _, c := range []struct{ name, data, want string }{
{"a class it does not know", `{"own":[{"id":"a","path":"${dir:d}","class":"precious"}]}`, "not one the mesh protects by"},
{"an access it does not have", `{"own":[{"id":"a","path":"${access:media}","class":"irreplaceable","redundancy":"an array"}]}`, "declares no access"},
{"irreplaceable and protected by nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":"none"}]}`, "protected one way or the other"},
{"a cache on redundancy", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","redundancy":"an array"}]}`, "a cache is not protected"},
{"kept with something not required", `{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}`, "does not require"},
{"a measure it does not know", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","measure":"du"}]}`, "walk, dataset or shallow"},
{"a cache expecting writes", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","active":"1d"}]}`, "only irreplaceable and valuable"},
{"a machine path", `{"own":[{"id":"a","path":"/srv/a","class":"cache"}]}`, "names no machine path"},
{"a directory it does not declare", `{"own":[{"id":"a","path":"${dir:nowhere}","class":"cache"}]}`, "declares no directory"},
{"a cache backed up", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","backup":"copy"}]}`, "disposable"},
{"a dump into nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":{"dump":"x","into":"b"}}]}`, "not one of the module's data items"},
{"a dump into itself", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":{"dump":"x","into":"a"}}]}`, "into itself"},
{"a bad duration", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","within":"soon"}]}`, "not a duration"},
{"the same id twice", `{"own":[{"id":"a","path":"${dir:d}","class":"cache"},{"id":"a","path":"${dir:d}/x","class":"cache"}]}`, "declared twice"},
{"own data of class none", `{"own":[{"id":"a","path":"${dir:d}","class":"none"}]}`, "only a provision"},
{"consumers of something not provided", `{"consumers":{"s3-bucket":{"class":"cache"}}}`, "does not provide"},
{"consumer data that lives nowhere", `{"consumers":{"q":{"class":"irreplaceable"}}}`, "says nowhere it lives"},
{"consumer data in an item less precious", `{"own":[{"id":"a","path":"${dir:d}","class":"rebuildable"}],"consumers":{"q":{"class":"irreplaceable","in":"a"}}}`, "which is only rebuildable"},
{"consumer data in something unknown", `{"consumers":{"q":{"class":"rebuildable","in":"elsewhere"}}}`, "neither one of its data items"},
} {
raw := `{"module":"m","version":"1","provides":["q"],"resources":[{"id":"d","type":"directory","mode":"0700"}],"data":` + c.data + `}`
_, err := ParseManifest([]byte(raw))
if err == nil || !strings.Contains(err.Error(), c.want) {
t.Errorf("%s: %v, want %q", c.name, err, c.want)
}
}
}
// The catalogue check: a granting provider says what it keeps, says it in one place, writes no backup
// line by hand, and every directory a container writes is declared (novox/hq ADR 0233).
func TestTheCatalogueCheckAsksEveryModuleForItsData(t *testing.T) {
unsaid := mustParse(t, `{"module":"q","version":"1","provides":[{"name":"queue","scope":"mesh"}],
"grants":{"queue":"${dir:g}"},"resources":[{"id":"g","type":"directory","mode":"0700"}]}`)
onTheOffer := mustParse(t, `{"module":"r","version":"1","provides":[{"name":"resolver","scope":"mesh","keeps-consumer-data":false}]}`)
byHand := mustParse(t, `{"module":"h","version":"1","resources":[{"id":"d","type":"directory","mode":"0700"}],
"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:d}"}]}`)
writes := mustParse(t, `{"module":"w","version":"1","resources":[{"id":"d","type":"directory","mode":"0700"},
{"id":"c","type":"directory","mode":"0700"},
{"id":"app","type":"container","image":"x@sha256:aa","volumes":["${dir:d}:/data","${dir:c}:/etc/app:ro","${dir:d}/f:/f"]}]}`)
problems := strings.Join(DataProblems(Shelf{"q": unsaid, "r": onTheOffer, "h": byHand, "w": writes}), "\n")
for _, want := range []string{
"q grants queue and does not say what it keeps",
"r says keeps-consumer-data on its offer",
"h's contribution 1 is a backup line written by hand",
`w mounts its directory "d" into a container to be written`,
} {
if !strings.Contains(problems, want) {
t.Errorf("the check does not say %q:\n%s", want, problems)
}
}
if strings.Contains(problems, `"c"`) {
t.Errorf("a read-only mount was taken for written data:\n%s", problems)
}
// The same module declaring the directory, as a cache, passes.
declared := mustParse(t, `{"module":"w","version":"1","data":{"own":[{"id":"d","path":"${dir:d}","class":"cache"}]},
"resources":[{"id":"d","type":"directory","mode":"0700"},
{"id":"app","type":"container","image":"x@sha256:aa","volumes":["${dir:d}:/data"]}]}`)
if p := DataProblems(Shelf{"w": declared}); len(p) > 0 {
t.Fatalf("a declared directory is still refused: %v", p)
}
}
// Consumer data said to live in a provision the module requires is protected only as well as that
// provision's provider protects its consumers' data.
func TestConsumerDataInARequiredProvisionIsJudgedAgainstItsProvider(t *testing.T) {
idp := mustParse(t, `{"module":"idp","version":"1","requires":["postgres-database"],
"provides":[{"name":"oidc-client","scope":"mesh"}],"grants":{"oidc-client":"${dir:g}"},
"resources":[{"id":"g","type":"directory","mode":"0700"}],
"data":{"consumers":{"oidc-client":{"class":"irreplaceable","in":"postgres-database"}}}}`)
cachy := mustParse(t, strings.Replace(declaredStore, `"consumers":{"postgres-database":{"class":"irreplaceable","in":"store"}}`,
`"consumers":{"postgres-database":{"class":"cache"}}`, 1))
if p := strings.Join(DataProblems(Shelf{"idp": idp, "pg": cachy}), "\n"); !strings.Contains(p, "not protected as irreplaceable") {
t.Fatalf("irreplaceable data kept in a cache passed: %s", p)
}
if p := DataProblems(Shelf{"idp": idp, "pg": mustParse(t, declaredStore)}); len(p) > 0 {
t.Fatalf("refused against a provider that keeps its consumers' data: %v", p)
}
}
// What a binding is sticky by follows the class (ADR 0232, generalised by 0233): a cache's consumers
// move freely, a store's do not; an older definition saying nothing still follows its grant.
func TestKeepingConsumerDataFollowsTheClass(t *testing.T) {
for class, keeps := range map[string]bool{"irreplaceable": true, "valuable": true, "rebuildable": true, "cache": false, "none": false} {
in := `,"in":"d"`
own := `"own":[{"id":"d","path":"${dir:d}","class":"irreplaceable"}],`
if !keeps {
in, own = "", ""
}
m := mustParse(t, `{"module":"p","version":"1","provides":[{"name":"q","scope":"mesh"}],"grants":{"q":"${dir:d}"},
"resources":[{"id":"d","type":"directory","mode":"0700"}],
"data":{`+own+`"consumers":{"q":{"class":"`+class+`"`+in+`}}}}`)
if m.KeepsConsumerData("q") != keeps {
t.Errorf("class %s keeps consumer data: %v, want %v", class, !keeps, keeps)
}
shelf := map[string]Manifest{"p": m}
if KeepsConsumerData(shelf, "q") != keeps {
t.Errorf("class %s: the provision by name keeps: %v", class, !keeps)
}
}
older := mustParse(t, `{"module":"p","version":"1","provides":[{"name":"q","scope":"mesh"}],"grants":{"q":"${dir:d}"},
"resources":[{"id":"d","type":"directory","mode":"0700"}]}`)
if !older.KeepsConsumerData("q") {
t.Fatal("an older definition that grants no longer keeps its consumers' data")
}
}
// The backup holder's lines are derived: the dump runs and the directory it writes into is kept,
// anything but a cache is copied by default — the standard nightly plan — an item on redundancy is not
// copied, and every item is listed with its class and protection for the holder to measure and watch:
// directories filled, a home directory filled from the machine, an operator's path from where the
// assignment placed it. A backup line still written by hand beside a data section is not placed.
func TestTheBackupHoldersLinesAreDerivedFromTheData(t *testing.T) {
pg := mustParse(t, declaredStore)
pg.Contributions = append(pg.Contributions, SeatContribution{Seat: BackupSeat, Kind: "backup", Content: "path /stale"})
agent := mustParse(t, `{"module":"agent","version":"1",
"data":{"own":[{"id":"home","path":"${dir:home}","class":"valuable"},{"id":"tmp","path":"${dir:home}/tmp","class":"cache"}]},
"resources":[{"id":"home","type":"directory","path":"${machine:account-home}/.agent","mode":"0700"}]}`)
media := mustParse(t, `{"module":"media","version":"1",
"accesses":[{"id":"films","mode":"read"}],
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"on a redundant array; no room for a copy","measure":"dataset"},
{"id":"meta","path":"${dir:meta}","class":"rebuildable"}]},
"resources":[{"id":"meta","type":"directory","mode":"0700"}]}`)
facts := map[string]string{"account-home": "/home/op"}
with := Rendering{Settings: SettingsBy{"media": {{From: "node", Values: map[string]any{AccessesSetting: map[string]any{"films": "/tank/films"}}}}}}
backup, err := seatContributions([]Manifest{pg, agent, media}, BackupSeat, BackupKindBackup, with, facts)
if err != nil {
t.Fatal(err)
}
want := "# agent\npath /home/op/.agent\n# media\npath /var/lib/media/meta\n# pg\nrun dump-all > /srv/store/dumps/all\npath /srv/store/dumps\n"
if backup != want {
t.Fatalf("backup lines:\n%s\nwant:\n%s", backup, want)
}
data, err := seatContributions([]Manifest{pg, agent, media}, BackupSeat, BackupKindData, with, facts)
if err != nil {
t.Fatal(err)
}
want = "# agent\nitem home valuable /home/op/.agent /home/op/.agent backup walk\nitem tmp cache /home/op/.agent/tmp - none walk\n" +
"# media\nitem films irreplaceable /tank/films - redundancy dataset\nitem meta rebuildable /var/lib/media/meta /var/lib/media/meta backup walk\n" +
"# pg\nitem store irreplaceable /srv/store /srv/store/dumps backup walk\nitem dumps rebuildable /srv/store/dumps - none walk\n"
if data != want {
t.Fatalf("data lines:\n%s\nwant:\n%s", data, want)
}
if _, err := seatContributions([]Manifest{agent}, BackupSeat, BackupKindData, Rendering{}, nil); err == nil {
t.Fatal("a home directory with no account on the machine reached the holder as a placeholder")
}
// What keeps something irreplaceable depends on the machine's backup holder — it backs it up or
// watches its array; valuable data alone is backed up where a holder is, and refused nowhere.
if !slices.Contains(DependsOn(media), BackupSeat) || !slices.Contains(DependsOn(pg), BackupSeat) ||
slices.Contains(DependsOn(agent), BackupSeat) {
t.Fatalf("depends: media %v, pg %v, agent %v", DependsOn(media), DependsOn(pg), DependsOn(agent))
}
if it, _ := media.DataItem("films"); it.OwnedByModule() || it.Protection() != "redundancy" {
t.Fatalf("an operator's path reads %+v", it)
}
}
// Data a consumer keeps with a provider as irreplaceable must be protected there: the check refuses a
// provider that keeps its consumers' data as a cache, or in an item with no protection.
func TestIrreplaceableDataKeptWithAProviderMustBeProtectedThere(t *testing.T) {
photos := mustParse(t, `{"module":"photos","version":"1","requires":["s3-bucket"],
"data":{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}}`)
store := func(backup string) Manifest {
return mustParse(t, `{"module":"objects","version":"1","provides":[{"name":"s3-bucket","scope":"mesh"}],
"grants":{"s3-bucket":"${dir:g}"},
"data":{"own":[{"id":"data","path":"${dir:data}","class":"valuable","backup":"`+backup+`"}],
"consumers":{"s3-bucket":{"class":"valuable","in":"data"}}},
"resources":[{"id":"g","type":"directory","mode":"0700"},{"id":"data","type":"directory","mode":"0700"}]}`)
}
if p := DataProblems(Shelf{"photos": photos, "objects": store("copy")}); len(p) > 0 {
t.Fatalf("a provider backing its consumers' data up was refused: %v", p)
}
if p := strings.Join(DataProblems(Shelf{"photos": photos, "objects": store("none")}), "\n"); !strings.Contains(p, "neither backed up") {
t.Fatalf("irreplaceable photos in an unprotected store passed: %s", p)
}
}
+1 -1
View File
@@ -576,7 +576,7 @@ func contributionsInto(resource map[string]any, m Manifest, modules []Manifest,
found := ofContributed.FindStringSubmatch(placeholder) found := ofContributed.FindStringSubmatch(placeholder)
first, second, _ := strings.Cut(found[2], ":") first, second, _ := strings.Cut(found[2], ":")
if found[1] == "contribution" { if found[1] == "contribution" {
placed, err := seatContributions(modules, first, second, with) placed, err := seatContributions(modules, first, second, with, facts)
if err != nil && failed == nil { if err != nil && failed == nil {
failed = err failed = err
} }
+7 -19
View File
@@ -239,25 +239,6 @@ func (m Manifest) IdentityBoundOf(provision string) IdentityBound {
return IdentityBound{} return IdentityBound{}
} }
// KeepsConsumerData is whether this module, providing a provision, keeps what each consumer writes
// there (novox/hq ADR 0232): whether a consumer bound to it is bound to its data.
//
// **What an offer says, it gets.** Unsaid, it follows from whether the provider grants each consumer
// a credential of its own: a provider that does makes an account for every consumer — a role and its
// database, a key and its bucket, a client — and what the consumer writes under that account stays
// with that provider. One that grants nothing keeps nothing of anybody's: the resolver, a CA, the
// artifact store each answer any consumer alike, and moving a consumer between two of them loses
// nothing.
func (m Manifest) KeepsConsumerData(provision string) bool {
for _, o := range m.Provides {
if o.Name == provision && o.KeepsConsumerData != nil {
return *o.KeepsConsumerData
}
}
_, grants := m.Grants[provision]
return grants
}
// KeepsConsumerData is whether a provision, by name, keeps its consumers' data across the // KeepsConsumerData is whether a provision, by name, keeps its consumers' data across the
// catalogue: true when any module providing it at the mesh's scope does. **A property of the name**, // catalogue: true when any module providing it at the mesh's scope does. **A property of the name**,
// as brokering is: two providers disagreeing would make the same binding sticky or free depending on // as brokering is: two providers disagreeing would make the same binding sticky or free depending on
@@ -466,6 +447,12 @@ type Manifest struct {
// (novox/hq ADR 0201). Not history — that is an event — and never a secret, sealed or not. // (novox/hq ADR 0201). Not history — that is an event — and never a secret, sealed or not.
State []StateDeclaration `json:"state,omitempty"` State []StateDeclaration `json:"state,omitempty"`
// Data is every kind of data this module keeps — its own, by directory, and what it keeps for
// its consumers, by provision — each with a class the mesh protects and watches it by (novox/hq
// ADR 0233). One list: the backup holder's lines, the bindings that do not move, what an
// unassignment retires and what the self-check measures are all derived from it.
Data *Data `json:"data,omitempty"`
// Reads are other modules' state this module reads and watches, each `<module>.<name>` // Reads are other modules' state this module reads and watches, each `<module>.<name>`
// (novox/hq ADR 0201). Read-only: only the owner's instances write. // (novox/hq ADR 0201). Read-only: only the owner's instances write.
Reads []string `json:"reads,omitempty"` Reads []string `json:"reads,omitempty"`
@@ -2007,6 +1994,7 @@ func ParseManifest(raw []byte) (Manifest, error) {
problems = append(problems, m.shellProblems()...) problems = append(problems, m.shellProblems()...)
problems = append(problems, m.contributionPlaceholderProblems()...) problems = append(problems, m.contributionPlaceholderProblems()...)
problems = append(problems, m.seatContributionProblems()...) problems = append(problems, m.seatContributionProblems()...)
problems = append(problems, m.dataProblems()...)
for i, r := range m.Resources { for i, r := range m.Resources {
id, _ := r["id"].(string) id, _ := r["id"].(string)
+27 -42
View File
@@ -23,46 +23,6 @@ const MessageBusSeat = "node-message-bus"
// BackupSeat is the machine's backups (novox/hq ADR 0214, to-be 43). // BackupSeat is the machine's backups (novox/hq ADR 0214, to-be 43).
const BackupSeat = "node-backup" const BackupSeat = "node-backup"
// storeProvisions are the provisions whose provider keeps its consumers' data (novox/hq ADR 0214):
// a module providing one must say how to back it up, or the data the mesh hands out is the data it
// cannot restore — issue 241's seven databases. A provision that holds nothing worth keeping (a
// route, a cache, a name) is not here; adding one is adding a store.
var storeProvisions = map[string]bool{
"postgres-database": true,
"mssql-database": true,
"mongodb-database": true,
"s3-bucket": true,
"influxdb-api": true,
"secret": true,
}
// CheckBackup is a store provider that contributes no backup (novox/hq ADR 0214, "How it is
// checked").
//
// **The catalogue check's, not parsing's.** A manifest already registered and running was written
// before the rule; refusing it on read would make the controller refuse the very providers whose
// data the rule protects. New definitions meet it in the catalogue check, where they are written.
func CheckBackup(m Manifest) []string {
backs := false
for _, c := range m.Contributions {
if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat && c.Kind == "backup" {
backs = true
}
}
if backs {
return nil
}
var problems []string
for _, o := range m.Provides {
if storeProvisions[o.Name] {
problems = append(problems, fmt.Sprintf(
"%s provides %s, which keeps its consumers' data, and contributes no backup to %s; a "+
"store says how its data is copied (novox/hq ADR 0214)", m.Module, o.Name, BackupSeat))
}
}
return problems
}
// SeatContribution is one piece of configuration a module gives a seat's holder to place. // SeatContribution is one piece of configuration a module gives a seat's holder to place.
type SeatContribution struct { type SeatContribution struct {
// Seat is the seat whose holder places it. // Seat is the seat whose holder places it.
@@ -188,7 +148,11 @@ func seatPlaceholderProblems(m Manifest, r map[string]any) []string {
// comment line naming it in the tool's grammar, and empty when nothing is contributed. A kind that // comment line naming it in the tool's grammar, and empty when nothing is contributed. A kind that
// takes directories has each contributor's `${dir:<id>}` filled with where that module's directory // takes directories has each contributor's `${dir:<id>}` filled with where that module's directory
// is on this machine (novox/hq to-be 43). // is on this machine (novox/hq to-be 43).
func seatContributions(modules []Manifest, seat, kind string, with Rendering) (string, error) { //
// A directory placed in a person's home is named `${machine:account-home}/…` (novox/hq to-be 29); in a
// kind that takes directories that is filled from the machine's facts as well, so the holder reads a
// path, never a placeholder (novox/hq ADR 0233: an item in the operator's home is backed up too).
func seatContributions(modules []Manifest, seat, kind string, with Rendering, facts map[string]string) (string, error) {
s, r, ok := receivable(seat, kind) s, r, ok := receivable(seat, kind)
if !ok { if !ok {
return "", nil return "", nil
@@ -197,7 +161,7 @@ func seatContributions(modules []Manifest, seat, kind string, with Rendering) (s
var b strings.Builder var b strings.Builder
for _, m := range inModuleOrder(modules) { for _, m := range inModuleOrder(modules) {
named := false named := false
for _, c := range m.Contributions { for _, c := range m.allContributions() {
if c.Kind != kind { if c.Kind != kind {
continue continue
} }
@@ -214,6 +178,27 @@ func seatContributions(modules []Manifest, seat, kind string, with Rendering) (s
if err != nil && failed == nil { if err != nil && failed == nil {
failed = err failed = err
} }
// An operator's path the module was given, as an item of data on it (novox/hq ADR 0233).
if accessRef.MatchString(filled) {
_, byID, err := accessesFor(m, with.Settings[m.Module])
if err == nil {
filled, err = accessFill(filled, byID, m.Module)
}
if err != nil && failed == nil {
failed = err
}
}
for _, key := range machineUsed(filled) {
value, has := facts[key]
if !has {
if failed == nil {
failed = fmt.Errorf("%s's %s for %s says ${machine:%s}, and this machine says %s",
m.Module, kind, s.Name, key, orNothing(namesOfFacts(facts)))
}
continue
}
filled = strings.ReplaceAll(filled, "${machine:"+key+"}", value)
}
content = filled content = filled
} }
b.WriteString(content) b.WriteString(content)
+6
View File
@@ -123,6 +123,12 @@ func contributedTo(m Manifest) []string {
out = append(out, s.Name) out = append(out, s.Name)
} }
} }
// And the backup holder for what is irreplaceable (novox/hq ADR 0233): it backs it up or watches the
// array it is on. What is valuable is backed up where a holder is — the standard plan — and makes
// no machine refuse it for want of one.
if m.NeedsBackupHolder() {
out = append(out, BackupSeat)
}
return out return out
} }
+4 -2
View File
@@ -256,8 +256,10 @@ var defaultSeats = append([]Seat{
// disasters. A module contributes `backup` lines — what to run to take a consistent copy, and // disasters. A module contributes `backup` lines — what to run to take a consistent copy, and
// which of its directories to keep. // which of its directories to keep.
{Name: BackupSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0214", {Name: BackupSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0214",
Serves: backupVerbs(), Serves: backupVerbs(),
Receives: []Receivable{{Kind: "backup", Comment: "#", Dirs: true}}}, // `backup` is what to run and which paths to keep; `data` every item a module declares, with its
// class, for the holder to measure (novox/hq ADR 0233). Both derived from modules' data sections.
Receives: []Receivable{{Kind: BackupKindBackup, Comment: "#", Dirs: true}, {Kind: BackupKindData, Comment: "#", Dirs: true}}},
// Deferred (novox/hq ADR 0121): renaming to mesh-private-network is a scope + server/client // Deferred (novox/hq ADR 0121): renaming to mesh-private-network is a scope + server/client
// model change, not a rename, so it stays until that is built. // model change, not a rename, so it stays until that is built.
{Name: "the-private-network", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, {Name: "the-private-network", Scope: ScopeNode, Decision: "novox/hq ADR 0110"},
+13 -2
View File
@@ -282,8 +282,10 @@ var ControllerVerbs = []Verb{
"cause": "with answer: the cause in a word (retire-waiting when absent)", "cause": "with answer: the cause in a word (retire-waiting when absent)",
}, nil)}, }, nil)},
{Name: "cleanup", Description: "Every consumer a provider holds retired — its age, its size where the " + {Name: "cleanup", Description: "Every consumer a provider holds retired — its age, its size where the " +
"backend knows, and why it was retired. With consumer (and node, module): the provider deletes that one " + "backend knows, and why it was retired — and every module's own data retired on its machine (ADR 0233). " +
"retired consumer — never an active one. With older-than: every retired consumer older than that many " + "With consumer (and node, module): the provider deletes that one retired consumer — never an active one; " +
"for a module's own retired item, consumer names the item and the machine's backup holder takes a last " +
"restore point of it, then deletes it. With older-than: every retired consumer older than that many " +
"days, listed; deleted only with confirm. Deleting is a hand act, which says why (novox/hq ADR 0230).", "days, listed; deleted only with confirm. Deleting is a hand act, which says why (novox/hq ADR 0230).",
Input: schema(map[string]string{ Input: schema(map[string]string{
"node": "with consumer: the machine the provider runs on", "node": "with consumer: the machine the provider runs on",
@@ -294,6 +296,15 @@ var ControllerVerbs = []Verb{
"why": "with consumer or older-than: why — required, and recorded in the hand-act log", "why": "with consumer or older-than: why — required, and recorded in the hand-act log",
"cause": "with consumer or older-than: the cause in a word (cleanup-waiting when absent)", "cause": "with consumer or older-than: the cause in a word (cleanup-waiting when absent)",
}, nil, "confirm")}, }, nil, "confirm")},
// The data every machine declares (novox/hq ADR 0233).
{Name: "data", Description: "Every item of data every machine declares, as the self-check last measured it: " +
"its class (irreplaceable, rebuildable, cache), where it is, its size, its newest write, its newest good " +
"backup and the bound on it — and what is retired: kept after its module left the machine, removed only by " +
"`cleanup delete` (novox/hq ADR 0233).",
Input: schema(map[string]string{
"machine": "one machine (optional)",
"retired": "\"true\": only what is retired",
}, nil, "retired")},
{Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " + {Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " +
"`builds` with that id follows it line by line, and the module is registered when the outcome comes.", "`builds` with that id follows it line by line, and the module is registered when the outcome comes.",
Input: schema(map[string]string{ Input: schema(map[string]string{
+316
View File
@@ -0,0 +1,316 @@
package inventory
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/novox/mesh-controller/internal/catalogue"
)
// The data a module declares, as the mesh found it on each machine (novox/hq ADR 0233).
//
// The self-check composes what every machine declares and asks its backup holder what it measured;
// this keeps both. An irreplaceable item a machine no longer declares — its module unassigned, or no
// longer pulled in — is retired here, not forgotten: kept, with when and why, until a person deletes
// it through `cleanup delete`.
// DeclaredData is one item a machine's composition declares now.
type DeclaredData struct {
Module, Item, Class string
// Owned is whether it is in the module's own directory: only that is the mesh's to retire.
Owned bool
// Protection is backup, redundancy, both ("backup+redundancy") or none.
Protection string
}
// Redundancy is the redundant storage an item is on, as its machine's holder read it.
type Redundancy struct {
Kind string `json:"kind"`
Where string `json:"where"`
Healthy *bool `json:"healthy"`
Said string `json:"said"`
}
// Measurement is what a machine's backup holder said of one item. Nil fields were not measured.
type Measurement struct {
Path string
Size *int64
LastWrite *time.Time
MeasuredAt *time.Time
LastBackup *time.Time
// Error is what the holder could not measure, when it could not.
Error string
// Precision is what the size is, as the holder said it: "exact", "dataset …", "partial: …", "no size: …".
Precision string
Redundancy *Redundancy
}
// DataRecord is one item as the mesh keeps it.
type DataRecord struct {
Machine, Module, Item, Class, Path string
Owned bool
Protection string
FirstSeen, DeclaredAt time.Time
MeasureError string
Precision string
Redundancy *Redundancy
Size *int64
LastWrite, MeasuredAt, LastBackup *time.Time
RetiredAt *time.Time
RetiredWhy string
DeletedAt *time.Time
DeletedBy, DeletedWhy string
}
// Comparable is whether a size is fit to compare with another: exact, or a dataset's own counters.
func Comparable(precision string) bool {
return precision == "" || precision == "exact" || strings.HasPrefix(precision, "dataset ")
}
// Dataset is the ZFS dataset a size was read from, when it was: what several items on one dataset share.
func Dataset(precision string) string {
rest, ok := strings.CutPrefix(precision, "dataset ")
if !ok {
return ""
}
name, _, _ := strings.Cut(rest, ":")
return name
}
// Retired is whether the item is retired and not deleted.
func (r DataRecord) Retired() bool { return r.RetiredAt != nil && r.DeletedAt == nil }
// Key names it in one string, the way conditions and verbs do: <machine>/<module>/<item>.
func (r DataRecord) Key() string { return r.Machine + "/" + r.Module + "/" + r.Item }
// DataChange is what recording a machine's data changed: what it retired and what came back.
type DataChange struct {
Retired, Reenabled []DataRecord
}
// readingEvery is how often a measurement is kept as a reading: the shrink is read over days, and a
// row every five minutes would say the same thing sixty times an hour.
const readingEvery = 55 * time.Minute
// readingsKept is how long readings are kept.
const readingsKept = 90 * 24 * time.Hour
// dataKey is one item's identity on one machine.
type dataKey struct{ module, item string }
// RecordData keeps what one machine declares now and what its holder measured, at now.
//
// **Only from a composition that worked.** The caller passes the declared set of a machine whose
// composition succeeded; an item missing from it is then really no longer declared. An irreplaceable
// one is retired — never deleted, never forgotten — and anything else is forgotten, because what is
// rebuildable or a cache is not the mesh's to keep track of once its module is gone. An item declared
// again comes back from retirement as it was.
func (i *Inventory) RecordData(ctx context.Context, machine string, declared []DeclaredData,
measured map[string]map[string]Measurement, why string, now time.Time) (DataChange, error) {
var change DataChange
tx, err := i.store.Pool().Begin(ctx)
if err != nil {
return change, err
}
defer tx.Rollback(ctx) //nolint:errcheck
existing := map[dataKey]DataRecord{}
rows, err := tx.Query(ctx, dataSelect+` where machine = $1`, machine)
if err != nil {
return change, err
}
for rows.Next() {
r, err := scanData(rows)
if err != nil {
rows.Close()
return change, err
}
existing[dataKey{r.Module, r.Item}] = r
}
rows.Close()
if err := rows.Err(); err != nil {
return change, err
}
seen := map[dataKey]bool{}
for _, d := range declared {
k := dataKey{d.Module, d.Item}
seen[k] = true
m := measured[d.Module][d.Item]
var red struct {
Kind, Where, Said *string
Healthy *bool
}
if r := m.Redundancy; r != nil {
red.Kind, red.Where, red.Said, red.Healthy = &r.Kind, &r.Where, &r.Said, r.Healthy
}
was, had := existing[k]
if had && was.Retired() {
change.Reenabled = append(change.Reenabled, was)
}
// Deleted and declared again is new data: it starts over.
fresh := !had || was.DeletedAt != nil
if _, err := tx.Exec(ctx, `
insert into data_item (machine, module, item, class, path, first_seen, declared_at,
size_bytes, last_write, measured_at, last_backup, owned, protection,
measure_error, redundancy_kind, redundancy_where, redundancy_healthy, redundancy_said,
precision)
values ($1, $2, $3, $4, $5, $6, $6, $7, $8, $9, $10, $12, $13, $14, $15, $16, $17, $18, $19)
on conflict (machine, module, item) do update set
class = excluded.class,
owned = excluded.owned,
protection = excluded.protection,
precision = case when excluded.measured_at is not null then excluded.precision else data_item.precision end,
size_bytes = case when excluded.measured_at is not null then excluded.size_bytes else data_item.size_bytes end,
measure_error = case when excluded.measured_at is not null then excluded.measure_error else data_item.measure_error end,
redundancy_kind = case when excluded.measured_at is not null then excluded.redundancy_kind else data_item.redundancy_kind end,
redundancy_where = case when excluded.measured_at is not null then excluded.redundancy_where else data_item.redundancy_where end,
redundancy_healthy = case when excluded.measured_at is not null then excluded.redundancy_healthy else data_item.redundancy_healthy end,
redundancy_said = case when excluded.measured_at is not null then excluded.redundancy_said else data_item.redundancy_said end,
path = case when excluded.path <> '' then excluded.path else data_item.path end,
first_seen = case when $11::boolean then excluded.first_seen else data_item.first_seen end,
declared_at = excluded.declared_at,
last_write = coalesce(excluded.last_write, data_item.last_write),
measured_at = coalesce(excluded.measured_at, data_item.measured_at),
last_backup = coalesce(excluded.last_backup, data_item.last_backup),
retired_at = null, retired_why = null,
deleted_at = null, deleted_by = null, deleted_why = null`,
machine, d.Module, d.Item, d.Class, m.Path, now, m.Size, m.LastWrite, m.MeasuredAt, m.LastBackup,
fresh, d.Owned, d.Protection, nullable(m.Error), red.Kind, red.Where, red.Healthy, red.Said,
nullable(m.Precision)); err != nil {
return change, err
}
if m.Size != nil && m.MeasuredAt != nil && Comparable(m.Precision) {
if _, err := tx.Exec(ctx, `
insert into data_reading (machine, module, item, at, size_bytes, last_write)
select $1, $2, $3, $4, $5, $6
where not exists (select 1 from data_reading
where machine = $1 and module = $2 and item = $3 and at > $4::timestamptz - $7::interval)`,
machine, d.Module, d.Item, *m.MeasuredAt, *m.Size, m.LastWrite,
fmt.Sprintf("%d seconds", int(readingEvery.Seconds()))); err != nil {
return change, err
}
}
}
for k, r := range existing {
if seen[k] || r.DeletedAt != nil || r.Retired() {
continue
}
if !catalogue.Retires(r.Class) || !r.Owned {
if _, err := tx.Exec(ctx, `delete from data_item where machine = $1 and module = $2 and item = $3`,
machine, k.module, k.item); err != nil {
return change, err
}
continue
}
if _, err := tx.Exec(ctx, `update data_item set retired_at = $4, retired_why = $5
where machine = $1 and module = $2 and item = $3`, machine, k.module, k.item, now, why); err != nil {
return change, err
}
at := now
r.RetiredAt, r.RetiredWhy = &at, why
change.Retired = append(change.Retired, r)
}
if _, err := tx.Exec(ctx, `delete from data_reading where at < $1`, now.Add(-readingsKept)); err != nil {
return change, err
}
return change, tx.Commit(ctx)
}
const dataSelect = `select machine, module, item, class, path, first_seen, declared_at, size_bytes, last_write,
measured_at, last_backup, retired_at, coalesce(retired_why, ''), deleted_at, coalesce(deleted_by, ''),
coalesce(deleted_why, ''), owned, protection, coalesce(measure_error, ''), redundancy_kind, redundancy_where,
redundancy_healthy, redundancy_said, coalesce(precision, '') from data_item`
func scanData(rows pgx.Row) (DataRecord, error) {
var r DataRecord
var kind, where, said *string
var healthy *bool
err := rows.Scan(&r.Machine, &r.Module, &r.Item, &r.Class, &r.Path, &r.FirstSeen, &r.DeclaredAt, &r.Size,
&r.LastWrite, &r.MeasuredAt, &r.LastBackup, &r.RetiredAt, &r.RetiredWhy, &r.DeletedAt, &r.DeletedBy,
&r.DeletedWhy, &r.Owned, &r.Protection, &r.MeasureError, &kind, &where, &healthy, &said, &r.Precision)
if err == nil && kind != nil {
r.Redundancy = &Redundancy{Kind: *kind, Healthy: healthy}
if where != nil {
r.Redundancy.Where = *where
}
if said != nil {
r.Redundancy.Said = *said
}
}
return r, err
}
func nullable(s string) *string {
if s == "" {
return nil
}
return &s
}
// Data is every item the mesh keeps, by machine, module and item.
func (i *Inventory) Data(ctx context.Context) ([]DataRecord, error) {
rows, err := i.store.Pool().Query(ctx, dataSelect+` order by machine, module, item`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []DataRecord
for rows.Next() {
r, err := scanData(rows)
if err != nil {
return nil, err
}
out = append(out, r)
}
return out, rows.Err()
}
// DataOf is one item.
func (i *Inventory) DataOf(ctx context.Context, machine, module, item string) (DataRecord, error) {
r, err := scanData(i.store.Pool().QueryRow(ctx, dataSelect+` where machine = $1 and module = $2 and item = $3`,
machine, module, item))
if errors.Is(err, pgx.ErrNoRows) {
return r, fmt.Errorf("the mesh knows no data %s of %s on %s", item, module, machine)
}
return r, err
}
// DataPeaks is each item's largest reading since a moment, keyed by DataRecord.Key.
func (i *Inventory) DataPeaks(ctx context.Context, since time.Time) (map[string]int64, error) {
rows, err := i.store.Pool().Query(ctx, `select machine, module, item, max(size_bytes) from data_reading
where at >= $1 group by machine, module, item`, since)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]int64{}
for rows.Next() {
var machine, module, item string
var peak int64
if err := rows.Scan(&machine, &module, &item, &peak); err != nil {
return nil, err
}
out[machine+"/"+module+"/"+item] = peak
}
return out, rows.Err()
}
// MarkDataDeleted records that a person deleted a retired item. Refused for an item not retired.
func (i *Inventory) MarkDataDeleted(ctx context.Context, machine, module, item, by, why string, at time.Time) error {
tag, err := i.store.Pool().Exec(ctx, `update data_item set deleted_at = $4, deleted_by = $5, deleted_why = $6
where machine = $1 and module = $2 and item = $3 and retired_at is not null and deleted_at is null`,
machine, module, item, at, by, why)
if err != nil {
return err
}
if tag.RowsAffected() == 0 {
return fmt.Errorf("%s of %s on %s is not retired: only retired data is deleted", item, module, machine)
}
return nil
}
+130
View File
@@ -0,0 +1,130 @@
package inventory
import (
"testing"
"time"
)
func size(n int64) *int64 { return &n }
func at(t time.Time) *time.Time { return &t }
// What a machine declares is kept with what its holder measured; an irreplaceable item it no longer
// declares — its module unassigned — is RETIRED and kept, never forgotten, and comes back as it was when
// declared again; anything else no longer declared is forgotten (novox/hq ADR 0233).
func TestAnUndeclaredIrreplaceableItemIsRetiredNotForgotten(t *testing.T) {
inv := fresh(t)
ctx := t.Context()
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
declared := []DeclaredData{
{Module: "home-assistant", Item: "config", Class: "irreplaceable", Owned: true},
{Module: "searxng", Item: "valkey", Class: "cache"},
{Module: "ollama", Item: "models", Class: "rebuildable"},
}
measured := map[string]map[string]Measurement{"home-assistant": {"config": {Path: "/var/lib/home-assistant/config",
Size: size(900 << 20), LastWrite: at(now.Add(-time.Minute)), MeasuredAt: at(now), LastBackup: at(now.Add(-6 * time.Hour))}}}
if _, err := inv.RecordData(ctx, "home", declared, measured, "", now); err != nil {
t.Fatal(err)
}
all, err := inv.Data(ctx)
if err != nil || len(all) != 3 {
t.Fatalf("%+v, %v", all, err)
}
// Unassigned: nothing of it is declared any more.
later := now.Add(time.Hour)
change, err := inv.RecordData(ctx, "home", nil, nil, "home-assistant unassigned", later)
if err != nil {
t.Fatal(err)
}
if len(change.Retired) != 1 || change.Retired[0].Item != "config" {
t.Fatalf("retired %+v", change.Retired)
}
all, err = inv.Data(ctx)
if err != nil {
t.Fatal(err)
}
if len(all) != 1 || !all[0].Retired() || all[0].Path != "/var/lib/home-assistant/config" ||
*all[0].Size != 900<<20 || all[0].RetiredWhy != "home-assistant unassigned" {
t.Fatalf("the irreplaceable item is not kept retired with where it is: %+v", all)
}
// A second run that still does not declare it changes nothing: retired once, not again.
change, err = inv.RecordData(ctx, "home", nil, nil, "again", later.Add(time.Hour))
if err != nil || len(change.Retired) != 0 {
t.Fatalf("retired twice: %+v, %v", change, err)
}
// Deleting needs it retired; assigned again it is not retired any more, and its history stays.
if err := inv.MarkDataDeleted(ctx, "home", "searxng", "valkey", "p", "w", later); err == nil {
t.Fatal("deleting something the mesh does not hold retired was recorded")
}
change, err = inv.RecordData(ctx, "home", declared[:1], nil, "", later.Add(2*time.Hour))
if err != nil || len(change.Reenabled) != 1 {
t.Fatalf("declared again: %+v, %v", change, err)
}
r, err := inv.DataOf(ctx, "home", "home-assistant", "config")
if err != nil || r.Retired() || !r.FirstSeen.Equal(now) || r.Size == nil {
t.Fatalf("declared again lost what was known: %+v, %v", r, err)
}
// Retired and then deleted by a person: recorded, never by dropping the row.
if _, err := inv.RecordData(ctx, "home", nil, nil, "unassigned again", later.Add(3*time.Hour)); err != nil {
t.Fatal(err)
}
if err := inv.MarkDataDeleted(ctx, "home", "home-assistant", "config", "jochen", "moved to the anchor", later.Add(4*time.Hour)); err != nil {
t.Fatal(err)
}
r, err = inv.DataOf(ctx, "home", "home-assistant", "config")
if err != nil || r.DeletedAt == nil || r.DeletedBy != "jochen" || r.Retired() {
t.Fatalf("a deletion is not on record: %+v, %v", r, err)
}
}
// A reading is kept at most once an hour, and the peak is read over the window asked.
func TestReadingsAreHourlyAndThePeakIsTheLargest(t *testing.T) {
inv := fresh(t)
ctx := t.Context()
now := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)
declared := []DeclaredData{{Module: "grafana", Item: "data", Class: "valuable", Owned: true}}
for i, s := range []int64{100, 120, 999, 130, 40} {
when := now.Add(time.Duration(i) * 20 * time.Minute)
if _, err := inv.RecordData(ctx, "ace", declared, map[string]map[string]Measurement{"grafana": {"data": {
Path: "/var/lib/grafana/data", Size: size(s), MeasuredAt: at(when)}}}, "", when); err != nil {
t.Fatal(err)
}
}
var n int
if err := inv.store.Pool().QueryRow(ctx, `select count(*) from data_reading`).Scan(&n); err != nil || n != 2 {
t.Fatalf("%d readings kept for five measurements over 80 minutes, want 2 (one an hour): %v", n, err)
}
peaks, err := inv.DataPeaks(ctx, now.Add(-time.Hour))
if err != nil || peaks["ace/grafana/data"] != 130 {
t.Fatalf("peak %v, %v", peaks, err)
}
r, _ := inv.DataOf(ctx, "ace", "grafana", "data")
if r.Size == nil || *r.Size != 40 {
t.Fatalf("the newest measurement is not the one on record: %+v", r)
}
}
// A partial walk's lower bound is kept as the newest measurement, said as partial, and never kept as a
// reading a shrink would be read against.
func TestAPartialMeasurementIsNeverAReading(t *testing.T) {
inv := fresh(t)
ctx := t.Context()
now := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)
declared := []DeclaredData{{Module: "big", Item: "data", Class: "valuable", Owned: true}}
if _, err := inv.RecordData(ctx, "home", declared, map[string]map[string]Measurement{"big": {"data": {
Path: "/srv/big", Size: size(5), MeasuredAt: at(now), Precision: "partial: stopped"}}}, "", now); err != nil {
t.Fatal(err)
}
var n int
if err := inv.store.Pool().QueryRow(ctx, `select count(*) from data_reading`).Scan(&n); err != nil || n != 0 {
t.Fatalf("%d readings from a partial measurement: %v", n, err)
}
r, err := inv.DataOf(ctx, "home", "big", "data")
if err != nil || r.Precision != "partial: stopped" || Comparable(r.Precision) {
t.Fatalf("%+v, %v", r, err)
}
}
@@ -0,0 +1,68 @@
-- The data a module declares, as the mesh found it on each machine (novox/hq ADR 0233).
--
-- A module's manifest says what data it keeps and of which class. The self-check asks each machine's
-- backup holder what it measured of every declared item — its size, its last write, its last good
-- backup — and keeps that here: so a shrink is read against what the item held before, an item a
-- machine no longer declares is still known to be there, and an empty copy of an item is told from a
-- full one somewhere else.
--
-- **Keyed by the machine's name, not a reference.** The data outlives the machine record, as a
-- binding's provider does (migration 0071): a machine leaving the mesh must not turn the record of
-- what it holds into nothing.
--
-- **Retired, never removed.** An irreplaceable or valuable item in a module's own directory that its
-- machine no longer declares — its module unassigned — is marked retired, with when and why, and stays
-- until a person deletes it through `cleanup delete` (ADR 0230); the deletion is recorded here too,
-- never by dropping the row. An item on an operator's path (an access) is never the mesh's to retire.
--
-- Numbered 0072, past 0071, the highest on main or any open branch when this was written.
create table data_item (
machine text not null,
module text not null,
item text not null,
class text not null,
-- Whether it is in the module's own directory (the mesh's to retire) or an operator's path, and
-- how it is protected: backup, redundancy, both, or none.
owned boolean not null default true,
protection text not null default '',
-- Where it is on the machine, as the backup holder last said; empty until one has.
path text not null default '',
first_seen timestamptz not null default now(),
-- When a composition of the machine last declared it.
declared_at timestamptz not null default now(),
-- The newest measurement: size in bytes, the newest write inside it, and when it was measured.
size_bytes bigint,
last_write timestamptz,
measured_at timestamptz,
-- The newest good backup that covers it.
last_backup timestamptz,
-- What the holder could not measure, when it could not: the path gone, a walk refused.
measure_error text,
-- What the newest size is: exact, a dataset's whole size, partial (a lower bound) or none.
precision text,
-- The redundant storage it is on, as the holder read it: zfs, md or btrfs, which pool or device,
-- whether it is healthy, and what it said.
redundancy_kind text,
redundancy_where text,
redundancy_healthy boolean,
redundancy_said text,
retired_at timestamptz,
retired_why text,
deleted_at timestamptz,
deleted_by text,
deleted_why text,
primary key (machine, module, item)
);
-- One row per measurement kept, at most one an hour per item, for ninety days: what a shrink is
-- read against. Only a comparable size is kept: exact, or a dataset's own counters — never a partial
-- walk's lower bound.
create table data_reading (
machine text not null,
module text not null,
item text not null,
at timestamptz not null,
size_bytes bigint not null,
last_write timestamptz,
primary key (machine, module, item, at)
);
+7 -4
View File
@@ -233,10 +233,13 @@ type RetirementRejected struct {
// RetirementState is a provider's answer to provisioner_retirement. // RetirementState is a provider's answer to provisioner_retirement.
type RetirementState struct { type RetirementState struct {
Resource string `json:"resource"` Resource string `json:"resource"`
Node string `json:"node"` Node string `json:"node"`
Held []string `json:"held"` Held []string `json:"held"`
StablePasses int `json:"stable_passes"` // HeldSizes is what each held consumer keeps on the backend, in bytes, where the backend can say
// (novox/hq ADR 0233): what an empty replacement of a consumer's data is told by.
HeldSizes map[string]int64 `json:"held_sizes,omitempty"`
StablePasses int `json:"stable_passes"`
// StableForSeconds is how long the same unasked set must hold as well (ADR 0230: ten minutes). // StableForSeconds is how long the same unasked set must hold as well (ADR 0230: ten minutes).
StableForSeconds int `json:"stable_for_seconds,omitempty"` StableForSeconds int `json:"stable_for_seconds,omitempty"`
// RetiresBy is "disable", or "mark-only" for a provider that cannot disable a consumer. // RetiresBy is "disable", or "mark-only" for a provider that cannot disable a consumer.