jochen
abf9125689
A walk over a large library every hour loads the array that protects it. An item now says how it is measured — a bounded daily walk, a dataset's counters, or its top level only — and a size that is a lower bound is kept as such and never read as a shrink.
792 lines
29 KiB
Go
792 lines
29 KiB
Go
package catalogue
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// A module declares the data it holds, and the mesh protects and watches it from that declaration
|
|
// (novox/hq ADR 0233).
|
|
//
|
|
// **One section, `data`, for every kind of data a module keeps:** its own, by directory — a store's
|
|
// files, a mail spool, an application's uploads — or by an operator's path it was given; what it keeps
|
|
// for its consumers, by the provision they reach it through; and what of its own lives with a
|
|
// provider, by the provision it requires. Each entry has a class — how precious it is — and, for its
|
|
// own data, how it is protected; everything the mesh does is derived from those, never written per
|
|
// module:
|
|
//
|
|
// - a binding to a consumer's data does not move (ADR 0232) where the provision's class keeps data;
|
|
// - the backup holder's lines are composed from it — a module no longer writes them by hand;
|
|
// - what an unassignment leaves behind of an irreplaceable or valuable item is retired, listed by
|
|
// `cleanup list` and deleted only by `cleanup delete` (ADR 0230);
|
|
// - the self-check measures every item and says when one shrinks, disappears, stops being written,
|
|
// goes without its backup, sits on a degraded array, or is replaced by an empty copy of itself —
|
|
// urgent for what is irreplaceable, a warning for what is valuable.
|
|
|
|
// The classes: how precious the data is. A fixed vocabulary, ranked by the operator (2026-10-06): a
|
|
// class is what the protections are derived from, so a new one is a decision, not a manifest's choice.
|
|
const (
|
|
// ClassIrreplaceable is what must never be lost: the operator names it (the media library, the
|
|
// photo sites' storage). Protected by a backup or by a declared redundancy — one is required —
|
|
// retired rather than removed, and every alert about it is urgent.
|
|
ClassIrreplaceable = "irreplaceable"
|
|
// ClassValuable is anybody's work that would be painful to lose: in the nightly backup by default,
|
|
// retired rather than removed, and every alert about it a warning.
|
|
ClassValuable = "valuable"
|
|
// ClassRebuildable can be made again from something kept elsewhere — a clone, an index, a
|
|
// download, a night's dump — at a cost in time, not in data. In the nightly backup by default;
|
|
// forgotten when its module goes, and never alerted on.
|
|
ClassRebuildable = "rebuildable"
|
|
// ClassCache may be emptied at any moment with nothing lost but speed: never backed up, never
|
|
// measured, never alerted on.
|
|
ClassCache = "cache"
|
|
// ClassNone is a provision that keeps nothing of its consumers' (consumers only): a resolver, a
|
|
// certificate authority, an artifact store.
|
|
ClassNone = "none"
|
|
)
|
|
|
|
// classRank orders the classes by how precious they are, so the stricter of two is chosen.
|
|
var classRank = map[string]int{ClassNone: 0, ClassCache: 1, ClassRebuildable: 2, ClassValuable: 3, ClassIrreplaceable: 4}
|
|
|
|
// StricterClass is the more precious of two classes.
|
|
func StricterClass(a, b string) string {
|
|
if classRank[b] > classRank[a] {
|
|
return b
|
|
}
|
|
return a
|
|
}
|
|
|
|
// Retires is whether a class's data is retired, not forgotten, when its machine no longer declares it.
|
|
func Retires(class string) bool { return class == ClassIrreplaceable || class == ClassValuable }
|
|
|
|
// Watched is whether a class's data raises conditions: irreplaceable and valuable.
|
|
func Watched(class string) bool { return Retires(class) }
|
|
|
|
// DefaultBackupWithin is how old the last good backup of an item may be before the self-check says so
|
|
// (novox/hq ADR 0214: a machine with data and no good backup in 48 hours).
|
|
const DefaultBackupWithin = 48 * time.Hour
|
|
|
|
// Data is a manifest's `data` section.
|
|
type Data struct {
|
|
// Own is the data this module keeps itself.
|
|
Own []DataItem `json:"own,omitempty"`
|
|
// Consumers is what it keeps for its consumers, per provision it grants.
|
|
Consumers map[string]ConsumerData `json:"consumers,omitempty"`
|
|
// KeptBy is what of its own lives with the provider of a provision it requires — its rows, its
|
|
// objects — and how precious that is. The provider's protections follow the stricter of its own
|
|
// class for its consumers and this.
|
|
KeptBy map[string]KeptData `json:"kept-by,omitempty"`
|
|
}
|
|
|
|
// DataItem is one piece of a module's own data.
|
|
type DataItem struct {
|
|
// ID names it within the module: what `data`, `cleanup` and a condition call it.
|
|
ID string `json:"id"`
|
|
// Path is one of the module's directories, `${dir:<id>}`, or an operator's path it was given,
|
|
// `${access:<id>}`, or a path beneath either. Never a machine path (novox/hq ADR 0112).
|
|
Path string `json:"path"`
|
|
Class string `json:"class"`
|
|
// Backup is how it is copied: "copy" (the holder reads it as it stands), "none", or a dump — a
|
|
// command that writes a consistent copy into another item, which is copied. Unsaid, anything but a
|
|
// cache is copied, unless it says it is protected by redundancy instead.
|
|
Backup *DataBackup `json:"backup,omitempty"`
|
|
// Redundancy says it is protected by the redundancy of the storage it lives on rather than by a
|
|
// copy, and why that is enough: the media library on an array there is no room to copy. The
|
|
// backup holder then watches that array, and a degraded one is said.
|
|
Redundancy string `json:"redundancy,omitempty"`
|
|
// Within is how old its last good backup may be; unsaid, DefaultBackupWithin.
|
|
Within string `json:"within,omitempty"`
|
|
// Measure is how the backup holder measures it: `walk` (the default — every file, at most daily and
|
|
// bounded, for small items), `dataset` (a ZFS dataset's own counters, hourly, nothing walked) or
|
|
// `shallow` (its top-level entries only, no size). A large item never says walk.
|
|
Measure string `json:"measure,omitempty"`
|
|
// Active is how long it may go unwritten before that is a fault — for data something is
|
|
// expected to write all the time. Unsaid, a quiet item is not a fault.
|
|
Active string `json:"active,omitempty"`
|
|
// Why is a line for the reviewer: why this class.
|
|
Why string `json:"why,omitempty"`
|
|
}
|
|
|
|
// ConsumerData is what a provider keeps for the consumers of one provision.
|
|
type ConsumerData struct {
|
|
Class string `json:"class"`
|
|
// In is where it lives: one of the module's own items (whose protection covers it), or a
|
|
// provision this module requires (whose provider keeps it, as its consumer). Unsaid only for none
|
|
// and cache.
|
|
In string `json:"in,omitempty"`
|
|
Why string `json:"why,omitempty"`
|
|
}
|
|
|
|
// KeptData is how precious what a module keeps with a provider is.
|
|
type KeptData struct {
|
|
Class string `json:"class"`
|
|
Why string `json:"why,omitempty"`
|
|
}
|
|
|
|
// DataBackup is how an item is copied.
|
|
type DataBackup struct {
|
|
Copy bool
|
|
None bool
|
|
// Dump is the command writing a consistent copy into the item Into.
|
|
Dump string
|
|
Into string
|
|
}
|
|
|
|
// UnmarshalJSON reads "copy", "none" or {"dump": "...", "into": "<item>"}.
|
|
func (b *DataBackup) UnmarshalJSON(raw []byte) error {
|
|
var word string
|
|
if err := json.Unmarshal(raw, &word); err == nil {
|
|
switch word {
|
|
case "copy":
|
|
*b = DataBackup{Copy: true}
|
|
case "none":
|
|
*b = DataBackup{None: true}
|
|
default:
|
|
return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}, not %q", word)
|
|
}
|
|
return nil
|
|
}
|
|
var full struct {
|
|
Dump string `json:"dump"`
|
|
Into string `json:"into"`
|
|
}
|
|
dec := json.NewDecoder(bytes.NewReader(raw))
|
|
dec.DisallowUnknownFields()
|
|
if err := dec.Decode(&full); err != nil {
|
|
return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}: %w", err)
|
|
}
|
|
*b = DataBackup{Dump: full.Dump, Into: full.Into}
|
|
return nil
|
|
}
|
|
|
|
// MarshalJSON writes it back in the form it was written.
|
|
func (b DataBackup) MarshalJSON() ([]byte, error) {
|
|
switch {
|
|
case b.Copy:
|
|
return json.Marshal("copy")
|
|
case b.None:
|
|
return json.Marshal("none")
|
|
}
|
|
return json.Marshal(struct {
|
|
Dump string `json:"dump"`
|
|
Into string `json:"into"`
|
|
}{b.Dump, b.Into})
|
|
}
|
|
|
|
// IsDump is whether the item is copied by a dump.
|
|
func (b *DataBackup) IsDump() bool { return b != nil && b.Dump != "" }
|
|
|
|
// BackedUp is whether the holder keeps restore points of this item: anything but a cache by default —
|
|
// the nightly backup is the standard plan — unless it says "none", or says it is protected by
|
|
// redundancy and says nothing of a backup.
|
|
func (it DataItem) BackedUp() bool {
|
|
switch {
|
|
case it.Backup == nil:
|
|
return it.Class != ClassCache && it.Redundancy == ""
|
|
case it.Backup.None:
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// Protection is how the item is protected, in one word: "backup", "redundancy", both joined by "+",
|
|
// or "none".
|
|
func (it DataItem) Protection() string {
|
|
var by []string
|
|
if it.BackedUp() {
|
|
by = append(by, "backup")
|
|
}
|
|
if it.Redundancy != "" {
|
|
by = append(by, "redundancy")
|
|
}
|
|
if len(by) == 0 {
|
|
return "none"
|
|
}
|
|
return strings.Join(by, "+")
|
|
}
|
|
|
|
// The ways an item is measured.
|
|
const (
|
|
MeasureWalk = "walk"
|
|
MeasureDataset = "dataset"
|
|
MeasureShallow = "shallow"
|
|
)
|
|
|
|
// MeasuredBy is how the item is measured, with the default applied.
|
|
func (it DataItem) MeasuredBy() string {
|
|
if it.Measure == "" {
|
|
return MeasureWalk
|
|
}
|
|
return it.Measure
|
|
}
|
|
|
|
// OwnedByModule is whether the item is in one of the module's own directories — the mesh's to retire —
|
|
// rather than an operator's path it was given, which the mesh never retires and never deletes.
|
|
func (it DataItem) OwnedByModule() bool { return strings.HasPrefix(it.Path, "${dir:") }
|
|
|
|
// BackupWithin is the item's bound on its last good backup.
|
|
func (it DataItem) BackupWithin() time.Duration {
|
|
if d, err := ParseDataDuration(it.Within); err == nil && d > 0 {
|
|
return d
|
|
}
|
|
return DefaultBackupWithin
|
|
}
|
|
|
|
// ActiveWithin is how long the item may go unwritten; zero when quiet is not a fault.
|
|
func (it DataItem) ActiveWithin() time.Duration {
|
|
d, _ := ParseDataDuration(it.Active)
|
|
return d
|
|
}
|
|
|
|
// ParseDataDuration reads "48h", "90m" or "7d"; empty is zero.
|
|
func ParseDataDuration(s string) (time.Duration, error) {
|
|
s = strings.TrimSpace(s)
|
|
if s == "" {
|
|
return 0, nil
|
|
}
|
|
if days, ok := strings.CutSuffix(s, "d"); ok {
|
|
n, err := strconv.Atoi(days)
|
|
if err != nil || n <= 0 {
|
|
return 0, fmt.Errorf("%q is not a number of days", s)
|
|
}
|
|
return time.Duration(n) * 24 * time.Hour, nil
|
|
}
|
|
d, err := time.ParseDuration(s)
|
|
if err != nil || d <= 0 {
|
|
return 0, fmt.Errorf("%q is not a duration (48h, 90m, 7d)", s)
|
|
}
|
|
return d, nil
|
|
}
|
|
|
|
// DataItems is the module's own data, in the order declared.
|
|
func (m Manifest) DataItems() []DataItem {
|
|
if m.Data == nil {
|
|
return nil
|
|
}
|
|
return m.Data.Own
|
|
}
|
|
|
|
// DataItem is one own item by id.
|
|
func (m Manifest) DataItem(id string) (DataItem, bool) {
|
|
for _, it := range m.DataItems() {
|
|
if it.ID == id {
|
|
return it, true
|
|
}
|
|
}
|
|
return DataItem{}, false
|
|
}
|
|
|
|
// ConsumerDataOf is what this module says it keeps for a provision's consumers, and whether it says.
|
|
func (m Manifest) ConsumerDataOf(provision string) (ConsumerData, bool) {
|
|
if m.Data == nil {
|
|
return ConsumerData{}, false
|
|
}
|
|
c, ok := m.Data.Consumers[provision]
|
|
return c, ok
|
|
}
|
|
|
|
// KeptByOf is how precious what this module keeps with a provision's provider is, and whether it says.
|
|
func (m Manifest) KeptByOf(provision string) (KeptData, bool) {
|
|
if m.Data == nil {
|
|
return KeptData{}, false
|
|
}
|
|
k, ok := m.Data.KeptBy[provision]
|
|
return k, ok
|
|
}
|
|
|
|
// keepsByClass is whether a class keeps something a binding must not move away from.
|
|
func keepsByClass(class string) bool {
|
|
return class == ClassIrreplaceable || class == ClassValuable || class == ClassRebuildable
|
|
}
|
|
|
|
// dataID is what an item's id may be: it is a token in a condition's key and a word on a line.
|
|
var dataID = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
|
|
|
|
// dataPathRef is an item's path: one of the module's directories or accesses, and optionally a path
|
|
// beneath it.
|
|
var dataPathRef = regexp.MustCompile(`^\$\{(dir|access):([a-z0-9][a-z0-9-]*)\}(/[^\s$]*)?$`)
|
|
|
|
// dataProblems is what is wrong with the `data` section itself, from the manifest alone: judged at
|
|
// registration as every other per-manifest problem is. Whether a module declares what it should is
|
|
// the catalogue check's (DataProblems), because a module already running was written before it.
|
|
func (m Manifest) dataProblems() []string {
|
|
if m.Data == nil {
|
|
return nil
|
|
}
|
|
var problems []string
|
|
say := func(format string, args ...any) {
|
|
problems = append(problems, fmt.Sprintf("%s's data: ", m.Module)+fmt.Sprintf(format, args...))
|
|
}
|
|
dirs := map[string]bool{}
|
|
for _, r := range m.Resources {
|
|
if fmt.Sprint(r["type"]) == "directory" {
|
|
dirs[fmt.Sprint(r["id"])] = true
|
|
}
|
|
}
|
|
accesses := map[string]bool{}
|
|
for _, a := range m.Accesses {
|
|
if a.ID != "" {
|
|
accesses[a.ID] = true
|
|
}
|
|
}
|
|
ids := map[string]DataItem{}
|
|
paths := map[string]string{}
|
|
for i, it := range m.Data.Own {
|
|
label := it.ID
|
|
if label == "" {
|
|
label = fmt.Sprintf("item %d", i+1)
|
|
}
|
|
if !dataID.MatchString(it.ID) {
|
|
say("%s has no usable id: lower-case letters, digits and dashes", label)
|
|
} else if _, twice := ids[it.ID]; twice {
|
|
say("%s is declared twice", it.ID)
|
|
}
|
|
ids[it.ID] = it
|
|
ref := dataPathRef.FindStringSubmatch(it.Path)
|
|
switch {
|
|
case ref == nil:
|
|
say("%s's path %q is not one of the module's directories or accesses: ${dir:<id>} or ${access:<id>}, "+
|
|
"or a path beneath one (a definition names no machine path, novox/hq ADR 0112)", label, it.Path)
|
|
case ref[1] == "dir" && !dirs[ref[2]]:
|
|
say("%s's path names ${dir:%s}, and %s declares no directory %q", label, ref[2], m.Module, ref[2])
|
|
case ref[1] == "access" && !accesses[ref[2]]:
|
|
say("%s's path names ${access:%s}, and %s declares no access %q", label, ref[2], m.Module, ref[2])
|
|
case strings.Contains(it.Path, ".."):
|
|
say("%s's path %q climbs out of its directory", label, it.Path)
|
|
}
|
|
if other, twice := paths[it.Path]; twice && it.Path != "" {
|
|
say("%s and %s name the same path %s", other, label, it.Path)
|
|
}
|
|
paths[it.Path] = label
|
|
switch it.Class {
|
|
case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache:
|
|
case ClassNone:
|
|
say("%s is class none, which only a provision keeping nothing of its consumers' is; own data "+
|
|
"that is disposable is cache", label)
|
|
default:
|
|
say("%s's class %q is not one the mesh protects by: irreplaceable, valuable, rebuildable or cache",
|
|
label, it.Class)
|
|
}
|
|
if it.Class == ClassIrreplaceable && !it.BackedUp() && strings.TrimSpace(it.Redundancy) == "" {
|
|
say("%s is irreplaceable and is neither backed up nor said to be protected by redundancy; the only "+
|
|
"copy of something is protected one way or the other (novox/hq ADR 0233) — copy it, dump it into "+
|
|
"another item, or say `redundancy` with why that is enough", label)
|
|
}
|
|
if it.Class == ClassCache && it.Backup != nil && !it.Backup.None {
|
|
say("%s is a cache and asks to be backed up; a cache is disposable, or it is not a cache", label)
|
|
}
|
|
if it.Class == ClassCache && it.Redundancy != "" {
|
|
say("%s is a cache and says it is protected by redundancy; a cache is not protected", label)
|
|
}
|
|
switch it.Measure {
|
|
case "", MeasureWalk, MeasureDataset, MeasureShallow:
|
|
default:
|
|
say("%s's measure %q is walk, dataset or shallow", label, it.Measure)
|
|
}
|
|
if _, err := ParseDataDuration(it.Within); err != nil {
|
|
say("%s's within: %v", label, err)
|
|
}
|
|
if _, err := ParseDataDuration(it.Active); err != nil {
|
|
say("%s's active: %v", label, err)
|
|
}
|
|
if it.Within != "" && !it.BackedUp() {
|
|
say("%s states within, and is not backed up", label)
|
|
}
|
|
if it.Active != "" && !Watched(it.Class) {
|
|
say("%s is %s and expects writes; only irreplaceable and valuable data is watched", label, it.Class)
|
|
}
|
|
}
|
|
// Dumps go into an item of the same module, declared, and not into themselves.
|
|
for _, it := range m.Data.Own {
|
|
if it.Backup == nil || it.Backup.Copy || it.Backup.None {
|
|
continue
|
|
}
|
|
switch into, ok := ids[it.Backup.Into]; {
|
|
case strings.TrimSpace(it.Backup.Dump) == "":
|
|
say("%s's backup names no dump command", it.ID)
|
|
case it.Backup.Into == "":
|
|
say("%s's dump says no item it writes into", it.ID)
|
|
case !ok:
|
|
say("%s's dump writes into %q, which is not one of the module's data items", it.ID, it.Backup.Into)
|
|
case into.ID == it.ID:
|
|
say("%s's dump writes into itself; a dump is copied from where it lands", it.ID)
|
|
case into.Class == ClassCache:
|
|
say("%s's dump writes into %s, a cache; what is copied must not be disposable", it.ID, into.ID)
|
|
}
|
|
if it.Backup.Dump != "" {
|
|
declared := map[string]string{}
|
|
for d := range dirs {
|
|
declared[d] = ""
|
|
}
|
|
if _, err := dirFill(it.Backup.Dump, declared, m.Module); err != nil {
|
|
say("%s's dump: %v", it.ID, err)
|
|
}
|
|
}
|
|
}
|
|
provided := map[string]bool{}
|
|
for _, o := range m.Provides {
|
|
provided[o.Name] = true
|
|
}
|
|
wants := map[string]bool{}
|
|
for _, w := range m.Wants() {
|
|
wants[w] = true
|
|
}
|
|
for _, provision := range sortedKeys(m.Data.Consumers) {
|
|
c := m.Data.Consumers[provision]
|
|
if !provided[provision] {
|
|
say("says what it keeps for the consumers of %q, which it does not provide", provision)
|
|
}
|
|
switch c.Class {
|
|
case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache, ClassNone:
|
|
default:
|
|
say("its consumers' %s is class %q; one of irreplaceable, valuable, rebuildable, cache or none", provision, c.Class)
|
|
}
|
|
switch {
|
|
case c.Class == ClassNone && c.In != "":
|
|
say("its consumers' %s keeps nothing and says where it is kept (%s)", provision, c.In)
|
|
case keepsByClass(c.Class) && c.In == "":
|
|
say("its consumers' %s is %s and says nowhere it lives: `in` names one of the module's items, or a "+
|
|
"provision it requires", provision, c.Class)
|
|
case c.In != "":
|
|
if own, ok := ids[c.In]; ok {
|
|
if c.Class == ClassIrreplaceable && !own.BackedUp() && own.Redundancy == "" {
|
|
say("its consumers' %s is irreplaceable and lives in %s, which is not protected", provision, c.In)
|
|
}
|
|
if classRank[own.Class] < classRank[c.Class] {
|
|
say("its consumers' %s is %s and lives in %s, which is only %s", provision, c.Class, c.In, own.Class)
|
|
}
|
|
} else if !wants[c.In] {
|
|
say("its consumers' %s lives in %q, which is neither one of its data items nor a provision it "+
|
|
"requires", provision, c.In)
|
|
}
|
|
}
|
|
}
|
|
for _, provision := range sortedKeys(m.Data.KeptBy) {
|
|
k := m.Data.KeptBy[provision]
|
|
if !wants[provision] {
|
|
say("says it keeps data with the provider of %q, which it does not require", provision)
|
|
}
|
|
switch k.Class {
|
|
case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache:
|
|
default:
|
|
say("what it keeps with %s is class %q; one of irreplaceable, valuable, rebuildable or cache", provision, k.Class)
|
|
}
|
|
}
|
|
return problems
|
|
}
|
|
|
|
// KeepsConsumerData is whether this module, providing a provision, keeps what each consumer writes
|
|
// there (novox/hq ADR 0232, ADR 0233): whether a consumer bound to it is bound to its data.
|
|
//
|
|
// **What the data section says, it gets**: irreplaceable, valuable or rebuildable keeps; cache and none
|
|
// do not — a cache lost in a move costs speed, not data. Before the section, an offer said it with
|
|
// `keeps-consumer-data`, which is still read; unsaid in both, a provider that grants each consumer a
|
|
// credential of its own keeps that consumer's data (ADR 0232 §1). The catalogue check refuses the
|
|
// unsaid case for a provider that grants (DataProblems), so the inference is what an older definition
|
|
// on the shelf gets, never a new one.
|
|
func (m Manifest) KeepsConsumerData(provision string) bool {
|
|
if c, ok := m.ConsumerDataOf(provision); ok {
|
|
return keepsByClass(c.Class)
|
|
}
|
|
for _, o := range m.Provides {
|
|
if o.Name == provision && o.KeepsConsumerData != nil {
|
|
return *o.KeepsConsumerData
|
|
}
|
|
}
|
|
_, grants := m.Grants[provision]
|
|
return grants
|
|
}
|
|
|
|
// NeedsBackupHolder is whether a module's data needs the machine's backup holder to be there: it keeps
|
|
// something irreplaceable — backed up by the holder, or protected by an array the holder watches. A
|
|
// module keeping only valuable or rebuildable data is backed up where a holder is, and refused nowhere
|
|
// for want of one: the standard plan, not a requirement.
|
|
func (m Manifest) NeedsBackupHolder() bool {
|
|
for _, it := range m.DataItems() {
|
|
if it.Class == ClassIrreplaceable {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// --- derived: the backup holder's lines ---------------------------------------------------------
|
|
|
|
// The node-backup seat's kinds (novox/hq to-be 43, ADR 0233): `backup` is what to run and which paths
|
|
// to keep, `data` every item with its class and protection, for the holder to measure and watch.
|
|
const (
|
|
BackupKindBackup = "backup"
|
|
BackupKindData = "data"
|
|
)
|
|
|
|
// derivedContributions is what a module's data section gives the backup holder: its backup lines and
|
|
// its items. Every item is listed, so a valuable one on a machine is measured whether or not it is
|
|
// copied; a cache is listed and not measured.
|
|
func (m Manifest) derivedContributions() []SeatContribution {
|
|
items := m.DataItems()
|
|
if len(items) == 0 {
|
|
return nil
|
|
}
|
|
var lines []string
|
|
kept := map[string]bool{}
|
|
keep := func(path string) {
|
|
if !kept[path] {
|
|
kept[path] = true
|
|
lines = append(lines, "path "+path)
|
|
}
|
|
}
|
|
for _, it := range items {
|
|
if !it.BackedUp() {
|
|
continue
|
|
}
|
|
if it.Backup.IsDump() {
|
|
lines = append(lines, "run "+strings.TrimSpace(it.Backup.Dump))
|
|
if into, ok := m.DataItem(it.Backup.Into); ok {
|
|
keep(into.Path)
|
|
}
|
|
continue
|
|
}
|
|
keep(it.Path)
|
|
}
|
|
var described []string
|
|
for _, it := range items {
|
|
covered := "-"
|
|
switch {
|
|
case it.Backup.IsDump():
|
|
if into, ok := m.DataItem(it.Backup.Into); ok {
|
|
covered = into.Path
|
|
}
|
|
case it.BackedUp():
|
|
covered = it.Path
|
|
}
|
|
described = append(described, fmt.Sprintf("item %s %s %s %s %s %s", it.ID, it.Class, it.Path, covered,
|
|
it.Protection(), it.MeasuredBy()))
|
|
}
|
|
var out []SeatContribution
|
|
if len(lines) > 0 {
|
|
out = append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindBackup, Content: strings.Join(lines, "\n") + "\n"})
|
|
}
|
|
return append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindData, Content: strings.Join(described, "\n") + "\n"})
|
|
}
|
|
|
|
// allContributions is every contribution a module makes to a seat's holder: what it wrote, and what
|
|
// its data section derives. **One list**: a module that declares its data has its backup lines
|
|
// composed from it, and a backup line it still writes by hand is not placed — the catalogue check
|
|
// refuses it — so the holder never copies two lists that disagree.
|
|
func (m Manifest) allContributions() []SeatContribution {
|
|
if m.Data == nil {
|
|
return m.Contributions
|
|
}
|
|
derived := m.derivedContributions()
|
|
out := make([]SeatContribution, 0, len(m.Contributions)+len(derived))
|
|
for _, c := range m.Contributions {
|
|
if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat {
|
|
continue
|
|
}
|
|
out = append(out, c)
|
|
}
|
|
return append(out, derived...)
|
|
}
|
|
|
|
// --- the catalogue check ------------------------------------------------------------------------
|
|
|
|
// DataProblems is what the catalogue check refuses about the data a module declares (novox/hq ADR
|
|
// 0233), judged over the manifests given together:
|
|
//
|
|
// - a provider that grants a provision says what it keeps for that provision's consumers;
|
|
// - nobody says it on the offer any more (`keeps-consumer-data`): the data section is the one place;
|
|
// - nobody writes a backup line by hand: it is derived from the data section;
|
|
// - a directory a container writes, mounted whole, is a data item of some class;
|
|
// - consumer data said to live in a required provision lives where that provision's provider keeps
|
|
// its consumers' data, as preciously, when the provider is given;
|
|
// - data a consumer keeps with a provider as irreplaceable is protected there, when the provider is
|
|
// given.
|
|
//
|
|
// **The check's, not registration's**, as ADR 0214's backup rule was: a module already on the shelf
|
|
// was written before the rule, and refusing it there would refuse the very providers whose data the
|
|
// rule protects. Each module meets it where it is written.
|
|
func DataProblems(shelf Shelf) []string {
|
|
var problems []string
|
|
for _, name := range shelfOrder(shelf) {
|
|
m := shelf[name]
|
|
for _, provision := range sortedKeys(m.Grants) {
|
|
if _, said := m.ConsumerDataOf(provision); !said {
|
|
problems = append(problems, fmt.Sprintf(
|
|
"%s grants %s and does not say what it keeps for its consumers: data.consumers.%s with a "+
|
|
"class — irreplaceable, valuable, rebuildable, cache, or none (novox/hq ADR 0233)", name, provision, provision))
|
|
}
|
|
}
|
|
for _, o := range m.Provides {
|
|
if o.KeepsConsumerData != nil {
|
|
problems = append(problems, fmt.Sprintf(
|
|
"%s says keeps-consumer-data on its offer of %s; it is said once, in data.consumers.%s, with a "+
|
|
"class (novox/hq ADR 0233)", name, o.Name, o.Name))
|
|
}
|
|
}
|
|
for i, c := range m.Contributions {
|
|
if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat {
|
|
problems = append(problems, fmt.Sprintf(
|
|
"%s's contribution %d is a backup line written by hand; backups are derived from the data "+
|
|
"section — declare the data, with its class and how it is protected (novox/hq ADR 0233)", name, i+1))
|
|
}
|
|
}
|
|
for _, dir := range writtenDirectories(m) {
|
|
if !coversDirectory(m, dir) {
|
|
problems = append(problems, fmt.Sprintf(
|
|
"%s mounts its directory %q into a container to be written, and declares no data in it: "+
|
|
"data.own with a class — cache if it is disposable (novox/hq ADR 0233)", name, dir))
|
|
}
|
|
}
|
|
if m.Data == nil {
|
|
continue
|
|
}
|
|
for _, provision := range sortedKeys(m.Data.Consumers) {
|
|
c := m.Data.Consumers[provision]
|
|
if c.In == "" {
|
|
continue
|
|
}
|
|
if _, own := m.DataItem(c.In); own {
|
|
continue
|
|
}
|
|
for _, pname := range shelfOrder(shelf) {
|
|
p := shelf[pname]
|
|
pc, said := p.ConsumerDataOf(c.In)
|
|
if !said || !providesName(p, c.In) {
|
|
continue
|
|
}
|
|
if classRank[pc.Class] < classRank[c.Class] {
|
|
problems = append(problems, fmt.Sprintf(
|
|
"%s keeps its consumers' %s, %s, in %s — and %s keeps its consumers' %s as %s, "+
|
|
"so it is not protected as %s", name, provision, c.Class, c.In, pname, c.In, pc.Class, c.Class))
|
|
}
|
|
}
|
|
}
|
|
for _, provision := range sortedKeys(m.Data.KeptBy) {
|
|
k := m.Data.KeptBy[provision]
|
|
if k.Class != ClassIrreplaceable {
|
|
continue
|
|
}
|
|
for _, pname := range shelfOrder(shelf) {
|
|
p := shelf[pname]
|
|
pc, said := p.ConsumerDataOf(provision)
|
|
if !said || !providesName(p, provision) {
|
|
continue
|
|
}
|
|
if !keepsByClass(pc.Class) {
|
|
problems = append(problems, fmt.Sprintf(
|
|
"%s keeps irreplaceable data with %s, and %s keeps its consumers' %s as %s — nothing of it is "+
|
|
"protected there", name, provision, pname, provision, pc.Class))
|
|
continue
|
|
}
|
|
if in, own := p.DataItem(pc.In); own && !in.BackedUp() && in.Redundancy == "" {
|
|
problems = append(problems, fmt.Sprintf(
|
|
"%s keeps irreplaceable data with %s, and %s keeps it in %s, which is neither backed up nor "+
|
|
"on declared redundancy", name, provision, pname, pc.In))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return problems
|
|
}
|
|
|
|
func providesName(m Manifest, provision string) bool {
|
|
for _, o := range m.Provides {
|
|
if o.Name == provision {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// writtenDirectories are the module's directories a container mounts whole and may write: a volume
|
|
// `${dir:<id>}:<target>` without `:ro`, or a directory stated by an absolute path mounted the same
|
|
// way. A file beneath a directory, or a read-only mount, is configuration the mesh wrote.
|
|
func writtenDirectories(m Manifest) []string {
|
|
absolute := map[string]string{}
|
|
for _, r := range m.Resources {
|
|
if fmt.Sprint(r["type"]) != "directory" {
|
|
continue
|
|
}
|
|
if p, ok := r["path"].(string); ok && strings.HasPrefix(p, "/") {
|
|
absolute[strings.TrimRight(p, "/")] = fmt.Sprint(r["id"])
|
|
}
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, r := range m.Resources {
|
|
if fmt.Sprint(r["type"]) != "container" {
|
|
continue
|
|
}
|
|
vols, _ := r["volumes"].([]any)
|
|
for _, v := range vols {
|
|
s, _ := v.(string)
|
|
mount := volumeMount.FindStringSubmatch(s)
|
|
if mount == nil || volumeReadOnly(mount[3]) {
|
|
continue
|
|
}
|
|
src := strings.TrimRight(mount[1], "/")
|
|
if ref := dirPlain.FindStringSubmatch(src); ref != nil {
|
|
seen[ref[1]] = true
|
|
} else if id, ok := absolute[src]; ok {
|
|
seen[id] = true
|
|
}
|
|
}
|
|
}
|
|
out := make([]string, 0, len(seen))
|
|
for id := range seen {
|
|
out = append(out, id)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// volumeMount is a container's volume, `<source>:<target>[:<options>]`, its source possibly a
|
|
// `${dir:<id>}` — whose own colon is not the separator.
|
|
var volumeMount = regexp.MustCompile(`^(\$\{(?:dir|access):[a-z0-9][a-z0-9-]*\}[^:]*|[^:]+):([^:]+)(?::(.*))?$`)
|
|
|
|
// volumeReadOnly is whether a volume's options mount it read-only.
|
|
func volumeReadOnly(options string) bool {
|
|
for _, o := range strings.Split(options, ",") {
|
|
if strings.TrimSpace(o) == "ro" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// dirPlain is a whole directory, `${dir:<id>}` and nothing after it.
|
|
var dirPlain = regexp.MustCompile(`^\$\{dir:([a-z0-9][a-z0-9-]*)\}$`)
|
|
|
|
// coversDirectory is whether a data item names the directory, a path within it, or a directory it
|
|
// is placed beneath.
|
|
func coversDirectory(m Manifest, dir string) bool {
|
|
parents := map[string]string{}
|
|
for _, r := range m.Resources {
|
|
if fmt.Sprint(r["type"]) != "directory" {
|
|
continue
|
|
}
|
|
if p, ok := r["path"].(string); ok {
|
|
if ref := dirRef.FindStringSubmatch(p); ref != nil && strings.HasPrefix(p, "${dir:") {
|
|
parents[fmt.Sprint(r["id"])] = ref[1]
|
|
}
|
|
}
|
|
}
|
|
for _, it := range m.DataItems() {
|
|
ref := dataPathRef.FindStringSubmatch(it.Path)
|
|
if ref == nil || ref[1] != "dir" {
|
|
continue
|
|
}
|
|
for d, hops := dir, 0; d != "" && hops < 4; d, hops = parents[d], hops+1 {
|
|
if ref[2] == d {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|