package catalogue import ( "bytes" "encoding/json" "fmt" "regexp" "strconv" "strings" "time" ) // A module declares the data it holds, and the mesh protects and watches it from that declaration // (novox/hq ADR 0233). // // **One section, `data`, for every kind of data a module keeps:** its own, by directory — a store's // files, a mail spool, an application's uploads — or by an operator's path it was given; what it keeps // for its consumers, by the provision they reach it through; and what of its own lives with a // provider, by the provision it requires. Each entry has a class — how precious it is — and, for its // own data, how it is protected; everything the mesh does is derived from those, never written per // module: // // - a binding to a consumer's data does not move (ADR 0232) where the provision's class keeps data; // - the backup holder's lines are composed from it — a module no longer writes them by hand; // - what an unassignment leaves behind of an irreplaceable or valuable item is retired, listed by // `cleanup list` and deleted only by `cleanup delete` (ADR 0230); // - the self-check measures every item and says when one shrinks, disappears, stops being written, // goes without its backup, sits on a degraded array, or is replaced by an empty copy of itself — // urgent for what is irreplaceable, a warning for what is valuable. // The classes: how precious the data is. A fixed vocabulary, ranked by the operator (2026-10-06): a // class is what the protections are derived from, so a new one is a decision, not a manifest's choice. const ( // ClassIrreplaceable is what must never be lost: the operator names it (the media library, the // photo sites' storage). Protected by a backup or by a declared redundancy — one is required — // retired rather than removed, and every alert about it is urgent. ClassIrreplaceable = "irreplaceable" // ClassValuable is anybody's work that would be painful to lose: in the nightly backup by default, // retired rather than removed, and every alert about it a warning. ClassValuable = "valuable" // ClassRebuildable can be made again from something kept elsewhere — a clone, an index, a // download, a night's dump — at a cost in time, not in data. In the nightly backup by default; // forgotten when its module goes, and never alerted on. ClassRebuildable = "rebuildable" // ClassCache may be emptied at any moment with nothing lost but speed: never backed up, never // measured, never alerted on. ClassCache = "cache" // ClassNone is a provision that keeps nothing of its consumers' (consumers only): a resolver, a // certificate authority, an artifact store. ClassNone = "none" ) // classRank orders the classes by how precious they are, so the stricter of two is chosen. var classRank = map[string]int{ClassNone: 0, ClassCache: 1, ClassRebuildable: 2, ClassValuable: 3, ClassIrreplaceable: 4} // StricterClass is the more precious of two classes. func StricterClass(a, b string) string { if classRank[b] > classRank[a] { return b } return a } // Retires is whether a class's data is retired, not forgotten, when its machine no longer declares it. func Retires(class string) bool { return class == ClassIrreplaceable || class == ClassValuable } // Watched is whether a class's data raises conditions: irreplaceable and valuable. func Watched(class string) bool { return Retires(class) } // DefaultBackupWithin is how old the last good backup of an item may be before the self-check says so // (novox/hq ADR 0214: a machine with data and no good backup in 48 hours). const DefaultBackupWithin = 48 * time.Hour // Data is a manifest's `data` section. type Data struct { // Own is the data this module keeps itself. Own []DataItem `json:"own,omitempty"` // Consumers is what it keeps for its consumers, per provision it grants. Consumers map[string]ConsumerData `json:"consumers,omitempty"` // KeptBy is what of its own lives with the provider of a provision it requires — its rows, its // objects — and how precious that is. The provider's protections follow the stricter of its own // class for its consumers and this. KeptBy map[string]KeptData `json:"kept-by,omitempty"` } // DataItem is one piece of a module's own data. type DataItem struct { // ID names it within the module: what `data`, `cleanup` and a condition call it. ID string `json:"id"` // Path is one of the module's directories, `${dir:}`, or an operator's path it was given, // `${access:}`, or a path beneath either. Never a machine path (novox/hq ADR 0112). Path string `json:"path"` Class string `json:"class"` // Backup is how it is copied: "copy" (the holder reads it as it stands), "none", or a dump — a // command that writes a consistent copy into another item, which is copied. Unsaid, anything but a // cache is copied, unless it says it is protected by redundancy instead. Backup *DataBackup `json:"backup,omitempty"` // Redundancy says it is protected by the redundancy of the storage it lives on rather than by a // copy, and why that is enough: the media library on an array there is no room to copy. The // backup holder then watches that array, and a degraded one is said. Redundancy string `json:"redundancy,omitempty"` // Within is how old its last good backup may be; unsaid, DefaultBackupWithin. Within string `json:"within,omitempty"` // Measure is how the backup holder measures it: `walk` (the default — every file, at most daily and // bounded, for small items), `dataset` (a ZFS dataset's own counters, hourly, nothing walked) or // `shallow` (its top-level entries only, no size). A large item never says walk. Measure string `json:"measure,omitempty"` // Active is how long it may go unwritten before that is a fault — for data something is // expected to write all the time. Unsaid, a quiet item is not a fault. Active string `json:"active,omitempty"` // Why is a line for the reviewer: why this class. Why string `json:"why,omitempty"` } // ConsumerData is what a provider keeps for the consumers of one provision. type ConsumerData struct { Class string `json:"class"` // In is where it lives: one of the module's own items (whose protection covers it), or a // provision this module requires (whose provider keeps it, as its consumer). Unsaid only for none // and cache. In string `json:"in,omitempty"` Why string `json:"why,omitempty"` } // KeptData is how precious what a module keeps with a provider is. type KeptData struct { Class string `json:"class"` Why string `json:"why,omitempty"` } // DataBackup is how an item is copied. type DataBackup struct { Copy bool None bool // Dump is the command writing a consistent copy into the item Into. Dump string Into string } // UnmarshalJSON reads "copy", "none" or {"dump": "...", "into": ""}. func (b *DataBackup) UnmarshalJSON(raw []byte) error { var word string if err := json.Unmarshal(raw, &word); err == nil { switch word { case "copy": *b = DataBackup{Copy: true} case "none": *b = DataBackup{None: true} default: return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}, not %q", word) } return nil } var full struct { Dump string `json:"dump"` Into string `json:"into"` } dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() if err := dec.Decode(&full); err != nil { return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}: %w", typedUnknown(err)) } *b = DataBackup{Dump: full.Dump, Into: full.Into} return nil } // MarshalJSON writes it back in the form it was written. func (b DataBackup) MarshalJSON() ([]byte, error) { switch { case b.Copy: return json.Marshal("copy") case b.None: return json.Marshal("none") } return json.Marshal(struct { Dump string `json:"dump"` Into string `json:"into"` }{b.Dump, b.Into}) } // IsDump is whether the item is copied by a dump. func (b *DataBackup) IsDump() bool { return b != nil && b.Dump != "" } // BackedUp is whether the holder keeps restore points of this item: anything but a cache by default — // the nightly backup is the standard plan — unless it says "none", or says it is protected by // redundancy and says nothing of a backup. func (it DataItem) BackedUp() bool { switch { case it.Backup == nil: return it.Class != ClassCache && it.Redundancy == "" case it.Backup.None: return false } return true } // Protection is how the item is protected, in one word: "backup", "redundancy", both joined by "+", // or "none". func (it DataItem) Protection() string { var by []string if it.BackedUp() { by = append(by, "backup") } if it.Redundancy != "" { by = append(by, "redundancy") } if len(by) == 0 { return "none" } return strings.Join(by, "+") } // The ways an item is measured. const ( MeasureWalk = "walk" MeasureDataset = "dataset" MeasureShallow = "shallow" ) // MeasuredBy is how the item is measured, with the default applied. func (it DataItem) MeasuredBy() string { if it.Measure == "" { return MeasureWalk } return it.Measure } // OwnedByModule is whether the item is in one of the module's own directories — the mesh's to retire — // rather than an operator's path it was given, which the mesh never retires and never deletes. func (it DataItem) OwnedByModule() bool { return strings.HasPrefix(it.Path, "${dir:") } // BackupWithin is the item's bound on its last good backup. func (it DataItem) BackupWithin() time.Duration { if d, err := ParseDataDuration(it.Within); err == nil && d > 0 { return d } return DefaultBackupWithin } // ActiveWithin is how long the item may go unwritten; zero when quiet is not a fault. func (it DataItem) ActiveWithin() time.Duration { d, _ := ParseDataDuration(it.Active) return d } // ParseDataDuration reads "48h", "90m" or "7d"; empty is zero. func ParseDataDuration(s string) (time.Duration, error) { s = strings.TrimSpace(s) if s == "" { return 0, nil } if days, ok := strings.CutSuffix(s, "d"); ok { n, err := strconv.Atoi(days) if err != nil || n <= 0 { return 0, fmt.Errorf("%q is not a number of days", s) } return time.Duration(n) * 24 * time.Hour, nil } d, err := time.ParseDuration(s) if err != nil || d <= 0 { return 0, fmt.Errorf("%q is not a duration (48h, 90m, 7d)", s) } return d, nil } // DataItems is the module's own data, in the order declared. func (m Manifest) DataItems() []DataItem { if m.Data == nil { return nil } return m.Data.Own } // DataItem is one own item by id. func (m Manifest) DataItem(id string) (DataItem, bool) { for _, it := range m.DataItems() { if it.ID == id { return it, true } } return DataItem{}, false } // ConsumerDataOf is what this module says it keeps for a provision's consumers, and whether it says. func (m Manifest) ConsumerDataOf(provision string) (ConsumerData, bool) { if m.Data == nil { return ConsumerData{}, false } c, ok := m.Data.Consumers[provision] return c, ok } // KeptByOf is how precious what this module keeps with a provision's provider is, and whether it says. func (m Manifest) KeptByOf(provision string) (KeptData, bool) { if m.Data == nil { return KeptData{}, false } k, ok := m.Data.KeptBy[provision] return k, ok } // keepsByClass is whether a class keeps something a binding must not move away from. func keepsByClass(class string) bool { return class == ClassIrreplaceable || class == ClassValuable || class == ClassRebuildable } // KeepsConsumerData is whether this module, providing a provision, keeps what each consumer writes // there (novox/hq ADR 0232, ADR 0233): whether a consumer bound to it is bound to its data. // // **What the data section says, it gets**: irreplaceable, valuable or rebuildable keeps; cache and none // do not — a cache lost in a move costs speed, not data. Before the section, an offer said it with // `keeps-consumer-data`, which is still read; unsaid in both, a provider that grants each consumer a // credential of its own keeps that consumer's data (ADR 0232 §1). The catalogue check refuses the // unsaid case for a provider that grants (DataProblems), so the inference is what an older definition // on the shelf gets, never a new one. func (m Manifest) KeepsConsumerData(provision string) bool { if c, ok := m.ConsumerDataOf(provision); ok { return keepsByClass(c.Class) } for _, o := range m.Provides { if o.Name == provision && o.KeepsConsumerData != nil { return *o.KeepsConsumerData } } _, grants := m.Grants[provision] return grants } // NeedsBackupHolder is whether a module's data needs the machine's backup holder to be there: it keeps // something irreplaceable — backed up by the holder, or protected by an array the holder watches. A // module keeping only valuable or rebuildable data is backed up where a holder is, and refused nowhere // for want of one: the standard plan, not a requirement. func (m Manifest) NeedsBackupHolder() bool { for _, it := range m.DataItems() { if it.Class == ClassIrreplaceable { return true } } return false } // --- derived: the backup holder's lines --------------------------------------------------------- // The node-backup seat's kinds (novox/hq to-be 43, ADR 0233): `backup` is what to run and which paths // to keep, `data` every item with its class and protection, for the holder to measure and watch. const ( BackupKindBackup = "backup" BackupKindData = "data" ) // derivedContributions is what a module's data section gives the backup holder: its backup lines and // its items. Every item is listed, so a valuable one on a machine is measured whether or not it is // copied; a cache is listed and not measured. func (m Manifest) derivedContributions() []SeatContribution { items := m.DataItems() if len(items) == 0 { return nil } var lines []string kept := map[string]bool{} keep := func(path string) { if !kept[path] { kept[path] = true lines = append(lines, "path "+path) } } for _, it := range items { if !it.BackedUp() { continue } if it.Backup.IsDump() { lines = append(lines, "run "+strings.TrimSpace(it.Backup.Dump)) if into, ok := m.DataItem(it.Backup.Into); ok { keep(into.Path) } continue } keep(it.Path) } var described []string for _, it := range items { covered := "-" switch { case it.Backup.IsDump(): if into, ok := m.DataItem(it.Backup.Into); ok { covered = into.Path } case it.BackedUp(): covered = it.Path } described = append(described, fmt.Sprintf("item %s %s %s %s %s %s", it.ID, it.Class, it.Path, covered, it.Protection(), it.MeasuredBy())) } var out []SeatContribution if len(lines) > 0 { out = append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindBackup, Content: strings.Join(lines, "\n") + "\n"}) } return append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindData, Content: strings.Join(described, "\n") + "\n"}) } // allContributions is every contribution a module makes to a seat's holder: what it wrote, and what // its data section derives. **One list**: a module that declares its data has its backup lines // composed from it, and a backup line it still writes by hand is not placed — the catalogue check // refuses it — so the holder never copies two lists that disagree. func (m Manifest) allContributions() []SeatContribution { if m.Data == nil { return m.Contributions } derived := m.derivedContributions() out := make([]SeatContribution, 0, len(m.Contributions)+len(derived)) for _, c := range m.Contributions { if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat { continue } out = append(out, c) } return append(out, derived...) } // --- the catalogue check ------------------------------------------------------------------------ // volumeMount is a container's volume, `:[:]`, its source possibly a // `${dir:}` — whose own colon is not the separator. var volumeMount = regexp.MustCompile(`^(\$\{(?:dir|access):[a-z0-9][a-z0-9-]*\}[^:]*|[^:]+):([^:]+)(?::(.*))?$`) // dirPlain is a whole directory, `${dir:}` and nothing after it. var dirPlain = regexp.MustCompile(`^\$\{dir:([a-z0-9][a-z0-9-]*)\}$`)