jochen
52af210e47
A module's data section says what it keeps and how precious it is; the backup holder's lines, binding stickiness, retirement on unassign and D13's conditions follow from it, so issue 273's empty replacement is said and an unassigned module's data is remembered, not forgotten.
767 lines
28 KiB
Go
767 lines
28 KiB
Go
package catalogue
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// A module declares the data it holds, and the mesh protects and watches it from that declaration
|
|
// (novox/hq ADR 0233).
|
|
//
|
|
// **One section, `data`, for every kind of data a module keeps:** its own, by directory — a store's
|
|
// files, a mail spool, an application's uploads — or by an operator's path it was given; what it keeps
|
|
// for its consumers, by the provision they reach it through; and what of its own lives with a
|
|
// provider, by the provision it requires. Each entry has a class — how precious it is — and, for its
|
|
// own data, how it is protected; everything the mesh does is derived from those, never written per
|
|
// module:
|
|
//
|
|
// - a binding to a consumer's data does not move (ADR 0232) where the provision's class keeps data;
|
|
// - the backup holder's lines are composed from it — a module no longer writes them by hand;
|
|
// - what an unassignment leaves behind of an irreplaceable or valuable item is retired, listed by
|
|
// `cleanup list` and deleted only by `cleanup delete` (ADR 0230);
|
|
// - the self-check measures every item and says when one shrinks, disappears, stops being written,
|
|
// goes without its backup, sits on a degraded array, or is replaced by an empty copy of itself —
|
|
// urgent for what is irreplaceable, a warning for what is valuable.
|
|
|
|
// The classes: how precious the data is. A fixed vocabulary, ranked by the operator (2026-10-06): a
|
|
// class is what the protections are derived from, so a new one is a decision, not a manifest's choice.
|
|
const (
|
|
// ClassIrreplaceable is what must never be lost: the operator names it (the media library, the
|
|
// photo sites' storage). Protected by a backup or by a declared redundancy — one is required —
|
|
// retired rather than removed, and every alert about it is urgent.
|
|
ClassIrreplaceable = "irreplaceable"
|
|
// ClassValuable is anybody's work that would be painful to lose: in the nightly backup by default,
|
|
// retired rather than removed, and every alert about it a warning.
|
|
ClassValuable = "valuable"
|
|
// ClassRebuildable can be made again from something kept elsewhere — a clone, an index, a
|
|
// download, a night's dump — at a cost in time, not in data. In the nightly backup by default;
|
|
// forgotten when its module goes, and never alerted on.
|
|
ClassRebuildable = "rebuildable"
|
|
// ClassCache may be emptied at any moment with nothing lost but speed: never backed up, never
|
|
// measured, never alerted on.
|
|
ClassCache = "cache"
|
|
// ClassNone is a provision that keeps nothing of its consumers' (consumers only): a resolver, a
|
|
// certificate authority, an artifact store.
|
|
ClassNone = "none"
|
|
)
|
|
|
|
// classRank orders the classes by how precious they are, so the stricter of two is chosen.
|
|
var classRank = map[string]int{ClassNone: 0, ClassCache: 1, ClassRebuildable: 2, ClassValuable: 3, ClassIrreplaceable: 4}
|
|
|
|
// StricterClass is the more precious of two classes.
|
|
func StricterClass(a, b string) string {
|
|
if classRank[b] > classRank[a] {
|
|
return b
|
|
}
|
|
return a
|
|
}
|
|
|
|
// Retires is whether a class's data is retired, not forgotten, when its machine no longer declares it.
|
|
func Retires(class string) bool { return class == ClassIrreplaceable || class == ClassValuable }
|
|
|
|
// Watched is whether a class's data raises conditions: irreplaceable and valuable.
|
|
func Watched(class string) bool { return Retires(class) }
|
|
|
|
// DefaultBackupWithin is how old the last good backup of an item may be before the self-check says so
|
|
// (novox/hq ADR 0214: a machine with data and no good backup in 48 hours).
|
|
const DefaultBackupWithin = 48 * time.Hour
|
|
|
|
// Data is a manifest's `data` section.
|
|
type Data struct {
|
|
// Own is the data this module keeps itself.
|
|
Own []DataItem `json:"own,omitempty"`
|
|
// Consumers is what it keeps for its consumers, per provision it grants.
|
|
Consumers map[string]ConsumerData `json:"consumers,omitempty"`
|
|
// KeptBy is what of its own lives with the provider of a provision it requires — its rows, its
|
|
// objects — and how precious that is. The provider's protections follow the stricter of its own
|
|
// class for its consumers and this.
|
|
KeptBy map[string]KeptData `json:"kept-by,omitempty"`
|
|
}
|
|
|
|
// DataItem is one piece of a module's own data.
|
|
type DataItem struct {
|
|
// ID names it within the module: what `data`, `cleanup` and a condition call it.
|
|
ID string `json:"id"`
|
|
// Path is one of the module's directories, `${dir:<id>}`, or an operator's path it was given,
|
|
// `${access:<id>}`, or a path beneath either. Never a machine path (novox/hq ADR 0112).
|
|
Path string `json:"path"`
|
|
Class string `json:"class"`
|
|
// Backup is how it is copied: "copy" (the holder reads it as it stands), "none", or a dump — a
|
|
// command that writes a consistent copy into another item, which is copied. Unsaid, anything but a
|
|
// cache is copied, unless it says it is protected by redundancy instead.
|
|
Backup *DataBackup `json:"backup,omitempty"`
|
|
// Redundancy says it is protected by the redundancy of the storage it lives on rather than by a
|
|
// copy, and why that is enough: the media library on an array there is no room to copy. The
|
|
// backup holder then watches that array, and a degraded one is said.
|
|
Redundancy string `json:"redundancy,omitempty"`
|
|
// Within is how old its last good backup may be; unsaid, DefaultBackupWithin.
|
|
Within string `json:"within,omitempty"`
|
|
// Active is how long it may go unwritten before that is a fault — for data something is
|
|
// expected to write all the time. Unsaid, a quiet item is not a fault.
|
|
Active string `json:"active,omitempty"`
|
|
// Why is a line for the reviewer: why this class.
|
|
Why string `json:"why,omitempty"`
|
|
}
|
|
|
|
// ConsumerData is what a provider keeps for the consumers of one provision.
|
|
type ConsumerData struct {
|
|
Class string `json:"class"`
|
|
// In is where it lives: one of the module's own items (whose protection covers it), or a
|
|
// provision this module requires (whose provider keeps it, as its consumer). Unsaid only for none
|
|
// and cache.
|
|
In string `json:"in,omitempty"`
|
|
Why string `json:"why,omitempty"`
|
|
}
|
|
|
|
// KeptData is how precious what a module keeps with a provider is.
|
|
type KeptData struct {
|
|
Class string `json:"class"`
|
|
Why string `json:"why,omitempty"`
|
|
}
|
|
|
|
// DataBackup is how an item is copied.
|
|
type DataBackup struct {
|
|
Copy bool
|
|
None bool
|
|
// Dump is the command writing a consistent copy into the item Into.
|
|
Dump string
|
|
Into string
|
|
}
|
|
|
|
// UnmarshalJSON reads "copy", "none" or {"dump": "...", "into": "<item>"}.
|
|
func (b *DataBackup) UnmarshalJSON(raw []byte) error {
|
|
var word string
|
|
if err := json.Unmarshal(raw, &word); err == nil {
|
|
switch word {
|
|
case "copy":
|
|
*b = DataBackup{Copy: true}
|
|
case "none":
|
|
*b = DataBackup{None: true}
|
|
default:
|
|
return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}, not %q", word)
|
|
}
|
|
return nil
|
|
}
|
|
var full struct {
|
|
Dump string `json:"dump"`
|
|
Into string `json:"into"`
|
|
}
|
|
dec := json.NewDecoder(bytes.NewReader(raw))
|
|
dec.DisallowUnknownFields()
|
|
if err := dec.Decode(&full); err != nil {
|
|
return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}: %w", err)
|
|
}
|
|
*b = DataBackup{Dump: full.Dump, Into: full.Into}
|
|
return nil
|
|
}
|
|
|
|
// MarshalJSON writes it back in the form it was written.
|
|
func (b DataBackup) MarshalJSON() ([]byte, error) {
|
|
switch {
|
|
case b.Copy:
|
|
return json.Marshal("copy")
|
|
case b.None:
|
|
return json.Marshal("none")
|
|
}
|
|
return json.Marshal(struct {
|
|
Dump string `json:"dump"`
|
|
Into string `json:"into"`
|
|
}{b.Dump, b.Into})
|
|
}
|
|
|
|
// IsDump is whether the item is copied by a dump.
|
|
func (b *DataBackup) IsDump() bool { return b != nil && b.Dump != "" }
|
|
|
|
// BackedUp is whether the holder keeps restore points of this item: anything but a cache by default —
|
|
// the nightly backup is the standard plan — unless it says "none", or says it is protected by
|
|
// redundancy and says nothing of a backup.
|
|
func (it DataItem) BackedUp() bool {
|
|
switch {
|
|
case it.Backup == nil:
|
|
return it.Class != ClassCache && it.Redundancy == ""
|
|
case it.Backup.None:
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// Protection is how the item is protected, in one word: "backup", "redundancy", both joined by "+",
|
|
// or "none".
|
|
func (it DataItem) Protection() string {
|
|
var by []string
|
|
if it.BackedUp() {
|
|
by = append(by, "backup")
|
|
}
|
|
if it.Redundancy != "" {
|
|
by = append(by, "redundancy")
|
|
}
|
|
if len(by) == 0 {
|
|
return "none"
|
|
}
|
|
return strings.Join(by, "+")
|
|
}
|
|
|
|
// OwnedByModule is whether the item is in one of the module's own directories — the mesh's to retire —
|
|
// rather than an operator's path it was given, which the mesh never retires and never deletes.
|
|
func (it DataItem) OwnedByModule() bool { return strings.HasPrefix(it.Path, "${dir:") }
|
|
|
|
// BackupWithin is the item's bound on its last good backup.
|
|
func (it DataItem) BackupWithin() time.Duration {
|
|
if d, err := ParseDataDuration(it.Within); err == nil && d > 0 {
|
|
return d
|
|
}
|
|
return DefaultBackupWithin
|
|
}
|
|
|
|
// ActiveWithin is how long the item may go unwritten; zero when quiet is not a fault.
|
|
func (it DataItem) ActiveWithin() time.Duration {
|
|
d, _ := ParseDataDuration(it.Active)
|
|
return d
|
|
}
|
|
|
|
// ParseDataDuration reads "48h", "90m" or "7d"; empty is zero.
|
|
func ParseDataDuration(s string) (time.Duration, error) {
|
|
s = strings.TrimSpace(s)
|
|
if s == "" {
|
|
return 0, nil
|
|
}
|
|
if days, ok := strings.CutSuffix(s, "d"); ok {
|
|
n, err := strconv.Atoi(days)
|
|
if err != nil || n <= 0 {
|
|
return 0, fmt.Errorf("%q is not a number of days", s)
|
|
}
|
|
return time.Duration(n) * 24 * time.Hour, nil
|
|
}
|
|
d, err := time.ParseDuration(s)
|
|
if err != nil || d <= 0 {
|
|
return 0, fmt.Errorf("%q is not a duration (48h, 90m, 7d)", s)
|
|
}
|
|
return d, nil
|
|
}
|
|
|
|
// DataItems is the module's own data, in the order declared.
|
|
func (m Manifest) DataItems() []DataItem {
|
|
if m.Data == nil {
|
|
return nil
|
|
}
|
|
return m.Data.Own
|
|
}
|
|
|
|
// DataItem is one own item by id.
|
|
func (m Manifest) DataItem(id string) (DataItem, bool) {
|
|
for _, it := range m.DataItems() {
|
|
if it.ID == id {
|
|
return it, true
|
|
}
|
|
}
|
|
return DataItem{}, false
|
|
}
|
|
|
|
// ConsumerDataOf is what this module says it keeps for a provision's consumers, and whether it says.
|
|
func (m Manifest) ConsumerDataOf(provision string) (ConsumerData, bool) {
|
|
if m.Data == nil {
|
|
return ConsumerData{}, false
|
|
}
|
|
c, ok := m.Data.Consumers[provision]
|
|
return c, ok
|
|
}
|
|
|
|
// KeptByOf is how precious what this module keeps with a provision's provider is, and whether it says.
|
|
func (m Manifest) KeptByOf(provision string) (KeptData, bool) {
|
|
if m.Data == nil {
|
|
return KeptData{}, false
|
|
}
|
|
k, ok := m.Data.KeptBy[provision]
|
|
return k, ok
|
|
}
|
|
|
|
// keepsByClass is whether a class keeps something a binding must not move away from.
|
|
func keepsByClass(class string) bool {
|
|
return class == ClassIrreplaceable || class == ClassValuable || class == ClassRebuildable
|
|
}
|
|
|
|
// dataID is what an item's id may be: it is a token in a condition's key and a word on a line.
|
|
var dataID = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
|
|
|
|
// dataPathRef is an item's path: one of the module's directories or accesses, and optionally a path
|
|
// beneath it.
|
|
var dataPathRef = regexp.MustCompile(`^\$\{(dir|access):([a-z0-9][a-z0-9-]*)\}(/[^\s$]*)?$`)
|
|
|
|
// dataProblems is what is wrong with the `data` section itself, from the manifest alone: judged at
|
|
// registration as every other per-manifest problem is. Whether a module declares what it should is
|
|
// the catalogue check's (DataProblems), because a module already running was written before it.
|
|
func (m Manifest) dataProblems() []string {
|
|
if m.Data == nil {
|
|
return nil
|
|
}
|
|
var problems []string
|
|
say := func(format string, args ...any) {
|
|
problems = append(problems, fmt.Sprintf("%s's data: ", m.Module)+fmt.Sprintf(format, args...))
|
|
}
|
|
dirs := map[string]bool{}
|
|
for _, r := range m.Resources {
|
|
if fmt.Sprint(r["type"]) == "directory" {
|
|
dirs[fmt.Sprint(r["id"])] = true
|
|
}
|
|
}
|
|
accesses := map[string]bool{}
|
|
for _, a := range m.Accesses {
|
|
if a.ID != "" {
|
|
accesses[a.ID] = true
|
|
}
|
|
}
|
|
ids := map[string]DataItem{}
|
|
paths := map[string]string{}
|
|
for i, it := range m.Data.Own {
|
|
label := it.ID
|
|
if label == "" {
|
|
label = fmt.Sprintf("item %d", i+1)
|
|
}
|
|
if !dataID.MatchString(it.ID) {
|
|
say("%s has no usable id: lower-case letters, digits and dashes", label)
|
|
} else if _, twice := ids[it.ID]; twice {
|
|
say("%s is declared twice", it.ID)
|
|
}
|
|
ids[it.ID] = it
|
|
ref := dataPathRef.FindStringSubmatch(it.Path)
|
|
switch {
|
|
case ref == nil:
|
|
say("%s's path %q is not one of the module's directories or accesses: ${dir:<id>} or ${access:<id>}, "+
|
|
"or a path beneath one (a definition names no machine path, novox/hq ADR 0112)", label, it.Path)
|
|
case ref[1] == "dir" && !dirs[ref[2]]:
|
|
say("%s's path names ${dir:%s}, and %s declares no directory %q", label, ref[2], m.Module, ref[2])
|
|
case ref[1] == "access" && !accesses[ref[2]]:
|
|
say("%s's path names ${access:%s}, and %s declares no access %q", label, ref[2], m.Module, ref[2])
|
|
case strings.Contains(it.Path, ".."):
|
|
say("%s's path %q climbs out of its directory", label, it.Path)
|
|
}
|
|
if other, twice := paths[it.Path]; twice && it.Path != "" {
|
|
say("%s and %s name the same path %s", other, label, it.Path)
|
|
}
|
|
paths[it.Path] = label
|
|
switch it.Class {
|
|
case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache:
|
|
case ClassNone:
|
|
say("%s is class none, which only a provision keeping nothing of its consumers' is; own data "+
|
|
"that is disposable is cache", label)
|
|
default:
|
|
say("%s's class %q is not one the mesh protects by: irreplaceable, valuable, rebuildable or cache",
|
|
label, it.Class)
|
|
}
|
|
if it.Class == ClassIrreplaceable && !it.BackedUp() && strings.TrimSpace(it.Redundancy) == "" {
|
|
say("%s is irreplaceable and is neither backed up nor said to be protected by redundancy; the only "+
|
|
"copy of something is protected one way or the other (novox/hq ADR 0233) — copy it, dump it into "+
|
|
"another item, or say `redundancy` with why that is enough", label)
|
|
}
|
|
if it.Class == ClassCache && it.Backup != nil && !it.Backup.None {
|
|
say("%s is a cache and asks to be backed up; a cache is disposable, or it is not a cache", label)
|
|
}
|
|
if it.Class == ClassCache && it.Redundancy != "" {
|
|
say("%s is a cache and says it is protected by redundancy; a cache is not protected", label)
|
|
}
|
|
if _, err := ParseDataDuration(it.Within); err != nil {
|
|
say("%s's within: %v", label, err)
|
|
}
|
|
if _, err := ParseDataDuration(it.Active); err != nil {
|
|
say("%s's active: %v", label, err)
|
|
}
|
|
if it.Within != "" && !it.BackedUp() {
|
|
say("%s states within, and is not backed up", label)
|
|
}
|
|
if it.Active != "" && !Watched(it.Class) {
|
|
say("%s is %s and expects writes; only irreplaceable and valuable data is watched", label, it.Class)
|
|
}
|
|
}
|
|
// Dumps go into an item of the same module, declared, and not into themselves.
|
|
for _, it := range m.Data.Own {
|
|
if it.Backup == nil || it.Backup.Copy || it.Backup.None {
|
|
continue
|
|
}
|
|
switch into, ok := ids[it.Backup.Into]; {
|
|
case strings.TrimSpace(it.Backup.Dump) == "":
|
|
say("%s's backup names no dump command", it.ID)
|
|
case it.Backup.Into == "":
|
|
say("%s's dump says no item it writes into", it.ID)
|
|
case !ok:
|
|
say("%s's dump writes into %q, which is not one of the module's data items", it.ID, it.Backup.Into)
|
|
case into.ID == it.ID:
|
|
say("%s's dump writes into itself; a dump is copied from where it lands", it.ID)
|
|
case into.Class == ClassCache:
|
|
say("%s's dump writes into %s, a cache; what is copied must not be disposable", it.ID, into.ID)
|
|
}
|
|
if it.Backup.Dump != "" {
|
|
declared := map[string]string{}
|
|
for d := range dirs {
|
|
declared[d] = ""
|
|
}
|
|
if _, err := dirFill(it.Backup.Dump, declared, m.Module); err != nil {
|
|
say("%s's dump: %v", it.ID, err)
|
|
}
|
|
}
|
|
}
|
|
provided := map[string]bool{}
|
|
for _, o := range m.Provides {
|
|
provided[o.Name] = true
|
|
}
|
|
wants := map[string]bool{}
|
|
for _, w := range m.Wants() {
|
|
wants[w] = true
|
|
}
|
|
for _, provision := range sortedKeys(m.Data.Consumers) {
|
|
c := m.Data.Consumers[provision]
|
|
if !provided[provision] {
|
|
say("says what it keeps for the consumers of %q, which it does not provide", provision)
|
|
}
|
|
switch c.Class {
|
|
case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache, ClassNone:
|
|
default:
|
|
say("its consumers' %s is class %q; one of irreplaceable, valuable, rebuildable, cache or none", provision, c.Class)
|
|
}
|
|
switch {
|
|
case c.Class == ClassNone && c.In != "":
|
|
say("its consumers' %s keeps nothing and says where it is kept (%s)", provision, c.In)
|
|
case keepsByClass(c.Class) && c.In == "":
|
|
say("its consumers' %s is %s and says nowhere it lives: `in` names one of the module's items, or a "+
|
|
"provision it requires", provision, c.Class)
|
|
case c.In != "":
|
|
if own, ok := ids[c.In]; ok {
|
|
if c.Class == ClassIrreplaceable && !own.BackedUp() && own.Redundancy == "" {
|
|
say("its consumers' %s is irreplaceable and lives in %s, which is not protected", provision, c.In)
|
|
}
|
|
if classRank[own.Class] < classRank[c.Class] {
|
|
say("its consumers' %s is %s and lives in %s, which is only %s", provision, c.Class, c.In, own.Class)
|
|
}
|
|
} else if !wants[c.In] {
|
|
say("its consumers' %s lives in %q, which is neither one of its data items nor a provision it "+
|
|
"requires", provision, c.In)
|
|
}
|
|
}
|
|
}
|
|
for _, provision := range sortedKeys(m.Data.KeptBy) {
|
|
k := m.Data.KeptBy[provision]
|
|
if !wants[provision] {
|
|
say("says it keeps data with the provider of %q, which it does not require", provision)
|
|
}
|
|
switch k.Class {
|
|
case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache:
|
|
default:
|
|
say("what it keeps with %s is class %q; one of irreplaceable, valuable, rebuildable or cache", provision, k.Class)
|
|
}
|
|
}
|
|
return problems
|
|
}
|
|
|
|
// KeepsConsumerData is whether this module, providing a provision, keeps what each consumer writes
|
|
// there (novox/hq ADR 0232, ADR 0233): whether a consumer bound to it is bound to its data.
|
|
//
|
|
// **What the data section says, it gets**: irreplaceable, valuable or rebuildable keeps; cache and none
|
|
// do not — a cache lost in a move costs speed, not data. Before the section, an offer said it with
|
|
// `keeps-consumer-data`, which is still read; unsaid in both, a provider that grants each consumer a
|
|
// credential of its own keeps that consumer's data (ADR 0232 §1). The catalogue check refuses the
|
|
// unsaid case for a provider that grants (DataProblems), so the inference is what an older definition
|
|
// on the shelf gets, never a new one.
|
|
func (m Manifest) KeepsConsumerData(provision string) bool {
|
|
if c, ok := m.ConsumerDataOf(provision); ok {
|
|
return keepsByClass(c.Class)
|
|
}
|
|
for _, o := range m.Provides {
|
|
if o.Name == provision && o.KeepsConsumerData != nil {
|
|
return *o.KeepsConsumerData
|
|
}
|
|
}
|
|
_, grants := m.Grants[provision]
|
|
return grants
|
|
}
|
|
|
|
// NeedsBackupHolder is whether a module's data needs the machine's backup holder to be there: it keeps
|
|
// something irreplaceable — backed up by the holder, or protected by an array the holder watches. A
|
|
// module keeping only valuable or rebuildable data is backed up where a holder is, and refused nowhere
|
|
// for want of one: the standard plan, not a requirement.
|
|
func (m Manifest) NeedsBackupHolder() bool {
|
|
for _, it := range m.DataItems() {
|
|
if it.Class == ClassIrreplaceable {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// --- derived: the backup holder's lines ---------------------------------------------------------
|
|
|
|
// The node-backup seat's kinds (novox/hq to-be 43, ADR 0233): `backup` is what to run and which paths
|
|
// to keep, `data` every item with its class and protection, for the holder to measure and watch.
|
|
const (
|
|
BackupKindBackup = "backup"
|
|
BackupKindData = "data"
|
|
)
|
|
|
|
// derivedContributions is what a module's data section gives the backup holder: its backup lines and
|
|
// its items. Every item is listed, so a valuable one on a machine is measured whether or not it is
|
|
// copied; a cache is listed and not measured.
|
|
func (m Manifest) derivedContributions() []SeatContribution {
|
|
items := m.DataItems()
|
|
if len(items) == 0 {
|
|
return nil
|
|
}
|
|
var lines []string
|
|
kept := map[string]bool{}
|
|
keep := func(path string) {
|
|
if !kept[path] {
|
|
kept[path] = true
|
|
lines = append(lines, "path "+path)
|
|
}
|
|
}
|
|
for _, it := range items {
|
|
if !it.BackedUp() {
|
|
continue
|
|
}
|
|
if it.Backup.IsDump() {
|
|
lines = append(lines, "run "+strings.TrimSpace(it.Backup.Dump))
|
|
if into, ok := m.DataItem(it.Backup.Into); ok {
|
|
keep(into.Path)
|
|
}
|
|
continue
|
|
}
|
|
keep(it.Path)
|
|
}
|
|
var described []string
|
|
for _, it := range items {
|
|
covered := "-"
|
|
switch {
|
|
case it.Backup.IsDump():
|
|
if into, ok := m.DataItem(it.Backup.Into); ok {
|
|
covered = into.Path
|
|
}
|
|
case it.BackedUp():
|
|
covered = it.Path
|
|
}
|
|
described = append(described, fmt.Sprintf("item %s %s %s %s %s", it.ID, it.Class, it.Path, covered, it.Protection()))
|
|
}
|
|
var out []SeatContribution
|
|
if len(lines) > 0 {
|
|
out = append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindBackup, Content: strings.Join(lines, "\n") + "\n"})
|
|
}
|
|
return append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindData, Content: strings.Join(described, "\n") + "\n"})
|
|
}
|
|
|
|
// allContributions is every contribution a module makes to a seat's holder: what it wrote, and what
|
|
// its data section derives. **One list**: a module that declares its data has its backup lines
|
|
// composed from it, and a backup line it still writes by hand is not placed — the catalogue check
|
|
// refuses it — so the holder never copies two lists that disagree.
|
|
func (m Manifest) allContributions() []SeatContribution {
|
|
if m.Data == nil {
|
|
return m.Contributions
|
|
}
|
|
derived := m.derivedContributions()
|
|
out := make([]SeatContribution, 0, len(m.Contributions)+len(derived))
|
|
for _, c := range m.Contributions {
|
|
if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat {
|
|
continue
|
|
}
|
|
out = append(out, c)
|
|
}
|
|
return append(out, derived...)
|
|
}
|
|
|
|
// --- the catalogue check ------------------------------------------------------------------------
|
|
|
|
// DataProblems is what the catalogue check refuses about the data a module declares (novox/hq ADR
|
|
// 0233), judged over the manifests given together:
|
|
//
|
|
// - a provider that grants a provision says what it keeps for that provision's consumers;
|
|
// - nobody says it on the offer any more (`keeps-consumer-data`): the data section is the one place;
|
|
// - nobody writes a backup line by hand: it is derived from the data section;
|
|
// - a directory a container writes, mounted whole, is a data item of some class;
|
|
// - consumer data said to live in a required provision lives where that provision's provider keeps
|
|
// its consumers' data, as preciously, when the provider is given;
|
|
// - data a consumer keeps with a provider as irreplaceable is protected there, when the provider is
|
|
// given.
|
|
//
|
|
// **The check's, not registration's**, as ADR 0214's backup rule was: a module already on the shelf
|
|
// was written before the rule, and refusing it there would refuse the very providers whose data the
|
|
// rule protects. Each module meets it where it is written.
|
|
func DataProblems(shelf Shelf) []string {
|
|
var problems []string
|
|
for _, name := range shelfOrder(shelf) {
|
|
m := shelf[name]
|
|
for _, provision := range sortedKeys(m.Grants) {
|
|
if _, said := m.ConsumerDataOf(provision); !said {
|
|
problems = append(problems, fmt.Sprintf(
|
|
"%s grants %s and does not say what it keeps for its consumers: data.consumers.%s with a "+
|
|
"class — irreplaceable, valuable, rebuildable, cache, or none (novox/hq ADR 0233)", name, provision, provision))
|
|
}
|
|
}
|
|
for _, o := range m.Provides {
|
|
if o.KeepsConsumerData != nil {
|
|
problems = append(problems, fmt.Sprintf(
|
|
"%s says keeps-consumer-data on its offer of %s; it is said once, in data.consumers.%s, with a "+
|
|
"class (novox/hq ADR 0233)", name, o.Name, o.Name))
|
|
}
|
|
}
|
|
for i, c := range m.Contributions {
|
|
if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat {
|
|
problems = append(problems, fmt.Sprintf(
|
|
"%s's contribution %d is a backup line written by hand; backups are derived from the data "+
|
|
"section — declare the data, with its class and how it is protected (novox/hq ADR 0233)", name, i+1))
|
|
}
|
|
}
|
|
for _, dir := range writtenDirectories(m) {
|
|
if !coversDirectory(m, dir) {
|
|
problems = append(problems, fmt.Sprintf(
|
|
"%s mounts its directory %q into a container to be written, and declares no data in it: "+
|
|
"data.own with a class — cache if it is disposable (novox/hq ADR 0233)", name, dir))
|
|
}
|
|
}
|
|
if m.Data == nil {
|
|
continue
|
|
}
|
|
for _, provision := range sortedKeys(m.Data.Consumers) {
|
|
c := m.Data.Consumers[provision]
|
|
if c.In == "" {
|
|
continue
|
|
}
|
|
if _, own := m.DataItem(c.In); own {
|
|
continue
|
|
}
|
|
for _, pname := range shelfOrder(shelf) {
|
|
p := shelf[pname]
|
|
pc, said := p.ConsumerDataOf(c.In)
|
|
if !said || !providesName(p, c.In) {
|
|
continue
|
|
}
|
|
if classRank[pc.Class] < classRank[c.Class] {
|
|
problems = append(problems, fmt.Sprintf(
|
|
"%s keeps its consumers' %s, %s, in %s — and %s keeps its consumers' %s as %s, "+
|
|
"so it is not protected as %s", name, provision, c.Class, c.In, pname, c.In, pc.Class, c.Class))
|
|
}
|
|
}
|
|
}
|
|
for _, provision := range sortedKeys(m.Data.KeptBy) {
|
|
k := m.Data.KeptBy[provision]
|
|
if k.Class != ClassIrreplaceable {
|
|
continue
|
|
}
|
|
for _, pname := range shelfOrder(shelf) {
|
|
p := shelf[pname]
|
|
pc, said := p.ConsumerDataOf(provision)
|
|
if !said || !providesName(p, provision) {
|
|
continue
|
|
}
|
|
if !keepsByClass(pc.Class) {
|
|
problems = append(problems, fmt.Sprintf(
|
|
"%s keeps irreplaceable data with %s, and %s keeps its consumers' %s as %s — nothing of it is "+
|
|
"protected there", name, provision, pname, provision, pc.Class))
|
|
continue
|
|
}
|
|
if in, own := p.DataItem(pc.In); own && !in.BackedUp() && in.Redundancy == "" {
|
|
problems = append(problems, fmt.Sprintf(
|
|
"%s keeps irreplaceable data with %s, and %s keeps it in %s, which is neither backed up nor "+
|
|
"on declared redundancy", name, provision, pname, pc.In))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return problems
|
|
}
|
|
|
|
func providesName(m Manifest, provision string) bool {
|
|
for _, o := range m.Provides {
|
|
if o.Name == provision {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// writtenDirectories are the module's directories a container mounts whole and may write: a volume
|
|
// `${dir:<id>}:<target>` without `:ro`, or a directory stated by an absolute path mounted the same
|
|
// way. A file beneath a directory, or a read-only mount, is configuration the mesh wrote.
|
|
func writtenDirectories(m Manifest) []string {
|
|
absolute := map[string]string{}
|
|
for _, r := range m.Resources {
|
|
if fmt.Sprint(r["type"]) != "directory" {
|
|
continue
|
|
}
|
|
if p, ok := r["path"].(string); ok && strings.HasPrefix(p, "/") {
|
|
absolute[strings.TrimRight(p, "/")] = fmt.Sprint(r["id"])
|
|
}
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, r := range m.Resources {
|
|
if fmt.Sprint(r["type"]) != "container" {
|
|
continue
|
|
}
|
|
vols, _ := r["volumes"].([]any)
|
|
for _, v := range vols {
|
|
s, _ := v.(string)
|
|
mount := volumeMount.FindStringSubmatch(s)
|
|
if mount == nil || volumeReadOnly(mount[3]) {
|
|
continue
|
|
}
|
|
src := strings.TrimRight(mount[1], "/")
|
|
if ref := dirPlain.FindStringSubmatch(src); ref != nil {
|
|
seen[ref[1]] = true
|
|
} else if id, ok := absolute[src]; ok {
|
|
seen[id] = true
|
|
}
|
|
}
|
|
}
|
|
out := make([]string, 0, len(seen))
|
|
for id := range seen {
|
|
out = append(out, id)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// volumeMount is a container's volume, `<source>:<target>[:<options>]`, its source possibly a
|
|
// `${dir:<id>}` — whose own colon is not the separator.
|
|
var volumeMount = regexp.MustCompile(`^(\$\{(?:dir|access):[a-z0-9][a-z0-9-]*\}[^:]*|[^:]+):([^:]+)(?::(.*))?$`)
|
|
|
|
// volumeReadOnly is whether a volume's options mount it read-only.
|
|
func volumeReadOnly(options string) bool {
|
|
for _, o := range strings.Split(options, ",") {
|
|
if strings.TrimSpace(o) == "ro" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// dirPlain is a whole directory, `${dir:<id>}` and nothing after it.
|
|
var dirPlain = regexp.MustCompile(`^\$\{dir:([a-z0-9][a-z0-9-]*)\}$`)
|
|
|
|
// coversDirectory is whether a data item names the directory, a path within it, or a directory it
|
|
// is placed beneath.
|
|
func coversDirectory(m Manifest, dir string) bool {
|
|
parents := map[string]string{}
|
|
for _, r := range m.Resources {
|
|
if fmt.Sprint(r["type"]) != "directory" {
|
|
continue
|
|
}
|
|
if p, ok := r["path"].(string); ok {
|
|
if ref := dirRef.FindStringSubmatch(p); ref != nil && strings.HasPrefix(p, "${dir:") {
|
|
parents[fmt.Sprint(r["id"])] = ref[1]
|
|
}
|
|
}
|
|
}
|
|
for _, it := range m.DataItems() {
|
|
ref := dataPathRef.FindStringSubmatch(it.Path)
|
|
if ref == nil || ref[1] != "dir" {
|
|
continue
|
|
}
|
|
for d, hops := dir, 0; d != "" && hops < 4; d, hops = parents[d], hops+1 {
|
|
if ref[2] == d {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|