mesh-sdk/go/catalogue/data.go

package catalogue

import (
	"bytes"
	"encoding/json"
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// The classes: how precious the data is. A fixed vocabulary, ranked by the operator (2026-10-06): a
// class is what the protections are derived from, so a new one is a decision, not a manifest's choice.
const (
	// ClassIrreplaceable is what must never be lost: the operator names it (the media library, the
	// photo sites' storage). Protected by a backup or by a declared redundancy — one is required —
	// retired rather than removed, and every alert about it is urgent.
	ClassIrreplaceable = "irreplaceable"
	// ClassValuable is anybody's work that would be painful to lose: in the nightly backup by default,
	// retired rather than removed, and every alert about it a warning.
	ClassValuable = "valuable"
	// ClassRebuildable can be made again from something kept elsewhere — a clone, an index, a
	// download, a night's dump — at a cost in time, not in data. In the nightly backup by default;
	// forgotten when its module goes, and never alerted on.
	ClassRebuildable = "rebuildable"
	// ClassCache may be emptied at any moment with nothing lost but speed: never backed up, never
	// measured, never alerted on.
	ClassCache = "cache"
	// ClassNone is a provision that keeps nothing of its consumers' (consumers only): a resolver, a
	// certificate authority, an artifact store.
	ClassNone = "none"
)

// classRank orders the classes by how precious they are, so the stricter of two is chosen.
var classRank = map[string]int{ClassNone: 0, ClassCache: 1, ClassRebuildable: 2, ClassValuable: 3, ClassIrreplaceable: 4}

// StricterClass is the more precious of two classes.
func StricterClass(a, b string) string {
	if classRank[b] > classRank[a] {
		return b
	}
	return a
}

// Retires is whether a class's data is retired, not forgotten, when its machine no longer declares it.
func Retires(class string) bool { return class == ClassIrreplaceable || class == ClassValuable }

// Watched is whether a class's data raises conditions: irreplaceable and valuable.
func Watched(class string) bool { return Retires(class) }

// DefaultBackupWithin is how old the last good backup of an item may be before the self-check says so
// (novox/hq ADR 0214: a machine with data and no good backup in 48 hours).
const DefaultBackupWithin = 48 * time.Hour

// Data is a manifest's `data` section.
type Data struct {
	// Own is the data this module keeps itself.
	Own []DataItem `json:"own,omitempty"`
	// Consumers is what it keeps for its consumers, per provision it grants.
	Consumers map[string]ConsumerData `json:"consumers,omitempty"`
	// KeptBy is what of its own lives with the provider of a provision it requires — its rows, its
	// objects — and how precious that is. The provider's protections follow the stricter of its own
	// class for its consumers and this.
	KeptBy map[string]KeptData `json:"kept-by,omitempty"`
}

// DataItem is one piece of a module's own data.
type DataItem struct {
	// ID names it within the module: what `data`, `cleanup` and a condition call it.
	ID string `json:"id"`
	// Path is one of the module's directories, `${dir:<id>}`, or an operator's path it was given,
	// `${access:<id>}`, or a path beneath either. Never a machine path (novox/hq ADR 0112).
	Path  string `json:"path"`
	Class string `json:"class"`
	// Backup is how it is copied: "copy" (the holder reads it as it stands), "none", or a dump — a
	// command that writes a consistent copy into another item, which is copied. Unsaid, anything but a
	// cache is copied, unless it says it is protected by redundancy instead.
	Backup *DataBackup `json:"backup,omitempty"`
	// Redundancy says it is protected by the redundancy of the storage it lives on rather than by a
	// copy, and why that is enough: the media library on an array there is no room to copy. The
	// backup holder then watches that array, and a degraded one is said.
	Redundancy string `json:"redundancy,omitempty"`
	// Within is how old its last good backup may be; unsaid, DefaultBackupWithin.
	Within string `json:"within,omitempty"`
	// Measure is how the backup holder measures it: `walk` (the default — every file, at most daily and
	// bounded, for small items), `dataset` (a ZFS dataset's own counters, hourly, nothing walked) or
	// `shallow` (its top-level entries only, no size). A large item never says walk.
	Measure string `json:"measure,omitempty"`
	// Active is how long it may go unwritten before that is a fault — for data something is
	// expected to write all the time. Unsaid, a quiet item is not a fault.
	Active string `json:"active,omitempty"`
	// Why is a line for the reviewer: why this class.
	Why string `json:"why,omitempty"`
}

// ConsumerData is what a provider keeps for the consumers of one provision.
type ConsumerData struct {
	Class string `json:"class"`
	// In is where it lives: one of the module's own items (whose protection covers it), or a
	// provision this module requires (whose provider keeps it, as its consumer). Unsaid only for none
	// and cache.
	In  string `json:"in,omitempty"`
	Why string `json:"why,omitempty"`
}

// KeptData is how precious what a module keeps with a provider is.
type KeptData struct {
	Class string `json:"class"`
	Why   string `json:"why,omitempty"`
}

// DataBackup is how an item is copied.
type DataBackup struct {
	Copy bool
	None bool
	// Dump is the command writing a consistent copy into the item Into.
	Dump string
	Into string
}

// UnmarshalJSON reads "copy", "none" or {"dump": "...", "into": "<item>"}.
func (b *DataBackup) UnmarshalJSON(raw []byte) error {
	var word string
	if err := json.Unmarshal(raw, &word); err == nil {
		switch word {
		case "copy":
			*b = DataBackup{Copy: true}
		case "none":
			*b = DataBackup{None: true}
		default:
			return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}, not %q", word)
		}
		return nil
	}
	var full struct {
		Dump string `json:"dump"`
		Into string `json:"into"`
	}
	dec := json.NewDecoder(bytes.NewReader(raw))
	dec.DisallowUnknownFields()
	if err := dec.Decode(&full); err != nil {
		return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}: %w", typedUnknown(err))
	}
	*b = DataBackup{Dump: full.Dump, Into: full.Into}
	return nil
}

// MarshalJSON writes it back in the form it was written.
func (b DataBackup) MarshalJSON() ([]byte, error) {
	switch {
	case b.Copy:
		return json.Marshal("copy")
	case b.None:
		return json.Marshal("none")
	}
	return json.Marshal(struct {
		Dump string `json:"dump"`
		Into string `json:"into"`
	}{b.Dump, b.Into})
}

// IsDump is whether the item is copied by a dump.
func (b *DataBackup) IsDump() bool { return b != nil && b.Dump != "" }

// BackedUp is whether the holder keeps restore points of this item: anything but a cache by default —
// the nightly backup is the standard plan — unless it says "none", or says it is protected by
// redundancy and says nothing of a backup.
func (it DataItem) BackedUp() bool {
	switch {
	case it.Backup == nil:
		return it.Class != ClassCache && it.Redundancy == ""
	case it.Backup.None:
		return false
	}
	return true
}

// Protection is how the item is protected, in one word: "backup", "redundancy", both joined by "+",
// or "none".
func (it DataItem) Protection() string {
	var by []string
	if it.BackedUp() {
		by = append(by, "backup")
	}
	if it.Redundancy != "" {
		by = append(by, "redundancy")
	}
	if len(by) == 0 {
		return "none"
	}
	return strings.Join(by, "+")
}

// The ways an item is measured.
const (
	MeasureWalk    = "walk"
	MeasureDataset = "dataset"
	MeasureShallow = "shallow"
)

// MeasuredBy is how the item is measured, with the default applied.
func (it DataItem) MeasuredBy() string {
	if it.Measure == "" {
		return MeasureWalk
	}
	return it.Measure
}

// BackupWithin is the item's bound on its last good backup.
func (it DataItem) BackupWithin() time.Duration {
	if d, err := ParseDataDuration(it.Within); err == nil && d > 0 {
		return d
	}
	return DefaultBackupWithin
}

// ActiveWithin is how long the item may go unwritten; zero when quiet is not a fault.
func (it DataItem) ActiveWithin() time.Duration {
	d, _ := ParseDataDuration(it.Active)
	return d
}

// ParseDataDuration reads "48h", "90m" or "7d"; empty is zero.
func ParseDataDuration(s string) (time.Duration, error) {
	s = strings.TrimSpace(s)
	if s == "" {
		return 0, nil
	}
	if days, ok := strings.CutSuffix(s, "d"); ok {
		n, err := strconv.Atoi(days)
		if err != nil || n <= 0 {
			return 0, fmt.Errorf("%q is not a number of days", s)
		}
		return time.Duration(n) * 24 * time.Hour, nil
	}
	d, err := time.ParseDuration(s)
	if err != nil || d <= 0 {
		return 0, fmt.Errorf("%q is not a duration (48h, 90m, 7d)", s)
	}
	return d, nil
}

// DataItems is the module's own data, in the order declared.
func (m Manifest) DataItems() []DataItem {
	if m.Data == nil {
		return nil
	}
	return m.Data.Own
}

// DataItem is one own item by id.
func (m Manifest) DataItem(id string) (DataItem, bool) {
	for _, it := range m.DataItems() {
		if it.ID == id {
			return it, true
		}
	}
	return DataItem{}, false
}

// ConsumerDataOf is what this module says it keeps for a provision's consumers, and whether it says.
func (m Manifest) ConsumerDataOf(provision string) (ConsumerData, bool) {
	if m.Data == nil {
		return ConsumerData{}, false
	}
	c, ok := m.Data.Consumers[provision]
	return c, ok
}

// keepsByClass is whether a class keeps something a binding must not move away from.
func keepsByClass(class string) bool {
	return class == ClassIrreplaceable || class == ClassValuable || class == ClassRebuildable
}

// KeepsConsumerData is whether this module, providing a provision, keeps what each consumer writes
// there (novox/hq ADR 0232, ADR 0233): whether a consumer bound to it is bound to its data.
//
// **What the data section says, it gets**: irreplaceable, valuable or rebuildable keeps; cache and none
// do not — a cache lost in a move costs speed, not data. Before the section, an offer said it with
// `keeps-consumer-data`, which is still read; unsaid in both, a provider that grants each consumer a
// credential of its own keeps that consumer's data (ADR 0232 §1). The catalogue check refuses the
// unsaid case for a provider that grants (DataProblems), so the inference is what an older definition
// on the shelf gets, never a new one.
func (m Manifest) KeepsConsumerData(provision string) bool {
	if c, ok := m.ConsumerDataOf(provision); ok {
		return keepsByClass(c.Class)
	}
	for _, o := range m.Provides {
		if o.Name == provision && o.KeepsConsumerData != nil {
			return *o.KeepsConsumerData
		}
	}
	_, grants := m.Grants[provision]
	return grants
}

// NeedsBackupHolder is whether a module's data needs the machine's backup holder to be there: it keeps
// something irreplaceable — backed up by the holder, or protected by an array the holder watches. A
// module keeping only valuable or rebuildable data is backed up where a holder is, and refused nowhere
// for want of one: the standard plan, not a requirement.
func (m Manifest) NeedsBackupHolder() bool {
	for _, it := range m.DataItems() {
		if it.Class == ClassIrreplaceable {
			return true
		}
	}
	return false
}

// The node-backup seat's kinds (novox/hq to-be 43, ADR 0233): `backup` is what to run and which paths
// to keep, `data` every item with its class and protection, for the holder to measure and watch.
const (
	BackupKindBackup = "backup"
	BackupKindData   = "data"
)

// derivedContributions is what a module's data section gives the backup holder: its backup lines and
// its items. Every item is listed, so a valuable one on a machine is measured whether or not it is
// copied; a cache is listed and not measured.
func (m Manifest) derivedContributions() []SeatContribution {
	items := m.DataItems()
	if len(items) == 0 {
		return nil
	}
	var lines []string
	kept := map[string]bool{}
	keep := func(path string) {
		if !kept[path] {
			kept[path] = true
			lines = append(lines, "path "+path)
		}
	}
	for _, it := range items {
		if !it.BackedUp() {
			continue
		}
		if it.Backup.IsDump() {
			lines = append(lines, "run "+strings.TrimSpace(it.Backup.Dump))
			if into, ok := m.DataItem(it.Backup.Into); ok {
				keep(into.Path)
			}
			continue
		}
		keep(it.Path)
	}
	var described []string
	for _, it := range items {
		covered := "-"
		switch {
		case it.Backup.IsDump():
			if into, ok := m.DataItem(it.Backup.Into); ok {
				covered = into.Path
			}
		case it.BackedUp():
			covered = it.Path
		}
		described = append(described, fmt.Sprintf("item %s %s %s %s %s %s", it.ID, it.Class, it.Path, covered,
			it.Protection(), it.MeasuredBy()))
	}
	var out []SeatContribution
	if len(lines) > 0 {
		out = append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindBackup, Content: strings.Join(lines, "\n") + "\n"})
	}
	return append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindData, Content: strings.Join(described, "\n") + "\n"})
}

// allContributions is every contribution a module makes to a seat's holder: what it wrote, and what
// its data section derives. **One list**: a module that declares its data has its backup lines
// composed from it, and a backup line it still writes by hand is not placed — the catalogue check
// refuses it — so the holder never copies two lists that disagree.
func (m Manifest) allContributions() []SeatContribution {
	if m.Data == nil {
		return m.Contributions
	}
	derived := m.derivedContributions()
	out := make([]SeatContribution, 0, len(m.Contributions)+len(derived))
	for _, c := range m.Contributions {
		if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat {
			continue
		}
		out = append(out, c)
	}
	return append(out, derived...)
}

// volumeMount is a container's volume, `<source>:<target>[:<options>]`, its source possibly a
// `${dir:<id>}` — whose own colon is not the separator.
var volumeMount = regexp.MustCompile(`^(\$\{(?:dir|access):[a-z0-9][a-z0-9-]*\}[^:]*|[^:]+):([^:]+)(?::(.*))?$`)

// dirPlain is a whole directory, `${dir:<id>}` and nothing after it.
var dirPlain = regexp.MustCompile(`^\$\{dir:([a-z0-9][a-z0-9-]*)\}$`)