Merge pull request 'A module declares the data it holds; protection and D13 derived from it (hq ADR 0233)' (#88) from feat/a-module-declares-the-data-it-holds into main
mesh/delivery held for a person: merged without a passing check: only a person decides that it goes on
mesh/delivery held for a person: merged without a passing check: only a person decides that it goes on
This commit was merged in pull request #88.
This commit is contained in:
@@ -224,6 +224,10 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
|||||||
for _, line := range unheldChange(shelf, node, assigned, left) {
|
for _, line := range unheldChange(shelf, node, assigned, left) {
|
||||||
answer += "\n " + line
|
answer += "\n " + line
|
||||||
}
|
}
|
||||||
|
// What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233).
|
||||||
|
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
|
||||||
|
answer += "\n " + line
|
||||||
|
}
|
||||||
return answer + blockedElsewhere(ctx, open, node), nil
|
return answer + blockedElsewhere(ctx, open, node), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -93,6 +93,15 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
failed += len(identities)
|
failed += len(identities)
|
||||||
|
|
||||||
|
// And the data each module keeps (novox/hq ADR 0233): a provider that grants says what it keeps for
|
||||||
|
// its consumers, a directory a container writes is declared, and no backup line is written by hand.
|
||||||
|
data := catalogue.DataProblems(shelf)
|
||||||
|
sort.Strings(data)
|
||||||
|
for _, p := range data {
|
||||||
|
fmt.Fprintln(out, p)
|
||||||
|
}
|
||||||
|
failed += len(data)
|
||||||
|
|
||||||
var names []string
|
var names []string
|
||||||
for name := range shelf {
|
for name := range shelf {
|
||||||
names = append(names, name)
|
names = append(names, name)
|
||||||
@@ -121,6 +130,14 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
|||||||
if len(m.Reads) > 0 {
|
if len(m.Reads) > 0 {
|
||||||
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
||||||
}
|
}
|
||||||
|
// The data it keeps, by class, so a reviewer sees what the mesh will protect and how.
|
||||||
|
if items := m.DataItems(); len(items) > 0 {
|
||||||
|
kept := make([]string, 0, len(items))
|
||||||
|
for _, it := range items {
|
||||||
|
kept = append(kept, it.ID+" ("+it.Class+")")
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, ", keeps %s", strings.Join(kept, ", "))
|
||||||
|
}
|
||||||
fmt.Fprintln(out)
|
fmt.Fprintln(out)
|
||||||
}
|
}
|
||||||
if failed > 0 {
|
if failed > 0 {
|
||||||
|
|||||||
@@ -0,0 +1,918 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module declares the data it holds, and the mesh protects and watches it from that declaration
|
||||||
|
// (novox/hq ADR 0233).
|
||||||
|
//
|
||||||
|
// The self-check's D13 composes what every machine declares, asks each machine's backup holder what
|
||||||
|
// it measured of every item — size, newest write, newest good backup, the redundant storage it is on —
|
||||||
|
// and keeps both. From that, and from what every provider says it holds for its consumers, it raises,
|
||||||
|
// each URGENT for what is irreplaceable and a WARNING for what is valuable (the operator's ranking):
|
||||||
|
//
|
||||||
|
// - `data-shrank`: an item holds less than half of its largest size in seven days, and at least
|
||||||
|
// shrinkFloor less; `data-missing`: its path is gone;
|
||||||
|
// - `empty-replacement`: an item, or a consumer's data at a provider, is less than half the size of a
|
||||||
|
// copy of the same thing kept elsewhere — on 2026-10-05 five applications ran for twenty hours on
|
||||||
|
// empty databases while their real ones sat on another machine (issue 273);
|
||||||
|
// - `data-held-twice` (warning): a consumer has active data at two providers and their sizes cannot
|
||||||
|
// be compared;
|
||||||
|
// - `data-quiet`: an item said to be written all the time has not been, within its bound;
|
||||||
|
// - `backup-stale`: an item's newest good backup is older than its bound, or there is none;
|
||||||
|
// - `array-degraded`: the redundant storage an item is on is not healthy, or cannot be read;
|
||||||
|
// `protection-missing`: an item said to be protected by redundancy is on storage that is not;
|
||||||
|
// - `cleanup-waiting` (warning): an item retired more than thirty days, waiting for a person.
|
||||||
|
//
|
||||||
|
// And it retires: an irreplaceable or valuable item in a module's own directory that its machine no
|
||||||
|
// longer declares — its module unassigned — is kept, marked retired with when and why, and listed by
|
||||||
|
// `cleanup list` until `cleanup delete` removes it. The node-engine never deletes a directory with
|
||||||
|
// anything in it; this is the record of what it kept. An operator's path is never retired or deleted.
|
||||||
|
|
||||||
|
// The condition kinds of D13.
|
||||||
|
const (
|
||||||
|
kindDataShrank = "data-shrank"
|
||||||
|
kindEmptyReplacement = "empty-replacement"
|
||||||
|
kindDataHeldTwice = "data-held-twice"
|
||||||
|
kindDataQuiet = "data-quiet"
|
||||||
|
kindBackupStale = "backup-stale"
|
||||||
|
kindDataUnmeasured = "data-unmeasured"
|
||||||
|
// kindDataMissing is a watched item whose path is gone.
|
||||||
|
kindDataMissing = "data-missing"
|
||||||
|
// kindArrayDegraded is redundant storage watched data is on that is not healthy, or cannot be read.
|
||||||
|
kindArrayDegraded = "array-degraded"
|
||||||
|
// kindProtectionMissing is an item said to be protected by redundancy, on storage that is not.
|
||||||
|
kindProtectionMissing = "protection-missing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// probeDataID is the self-check's id for this probe.
|
||||||
|
const probeDataID = "D13"
|
||||||
|
|
||||||
|
// The bounds the findings are read against.
|
||||||
|
var (
|
||||||
|
// shrinkWindow is how far back the largest size is looked for.
|
||||||
|
shrinkWindow = 7 * 24 * time.Hour
|
||||||
|
// shrinkFloor is the least loss that is worth saying: two empty databases differ by a few
|
||||||
|
// megabytes, and half of almost nothing is noise.
|
||||||
|
shrinkFloor int64 = 16 << 20
|
||||||
|
// dataAsk is how long one machine's holder, or one provider, is given to answer.
|
||||||
|
dataAsk = 8 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// keyOfItem is one item's condition id: its machine, module and item.
|
||||||
|
func keyOfItem(machine, module, item string) string { return machine + "." + module + "." + item }
|
||||||
|
|
||||||
|
// holderAnswer is what a node-backup holder's `backed-up` says of one module (ADR 0233 adds Data).
|
||||||
|
type holderAnswer struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Data []holderItem `json:"data"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// holderItem is one item as the holder measured it.
|
||||||
|
type holderItem struct {
|
||||||
|
Item string `json:"item"`
|
||||||
|
Class string `json:"class"`
|
||||||
|
Path string `json:"path"`
|
||||||
|
SizeBytes *int64 `json:"size_bytes"`
|
||||||
|
LastWrite *time.Time `json:"last_write"`
|
||||||
|
MeasuredAt *time.Time `json:"measured_at"`
|
||||||
|
LastBackup *time.Time `json:"last_backup"`
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
// Precision is what the size is: exact, a dataset's, partial, or none (ADR 0233).
|
||||||
|
Precision string `json:"precision,omitempty"`
|
||||||
|
// Redundancy is the redundant storage the item is on, where the holder could tell (ADR 0233).
|
||||||
|
Redundancy *inventory.Redundancy `json:"redundancy,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// readHolder reads a holder's answer into measurements by module and item.
|
||||||
|
func readHolder(raw json.RawMessage) (map[string]map[string]inventory.Measurement, error) {
|
||||||
|
var modules []holderAnswer
|
||||||
|
if err := json.Unmarshal(raw, &modules); err != nil {
|
||||||
|
return nil, fmt.Errorf("its answer is not readable: %w", err)
|
||||||
|
}
|
||||||
|
out := map[string]map[string]inventory.Measurement{}
|
||||||
|
for _, m := range modules {
|
||||||
|
for _, it := range m.Data {
|
||||||
|
if out[m.Module] == nil {
|
||||||
|
out[m.Module] = map[string]inventory.Measurement{}
|
||||||
|
}
|
||||||
|
out[m.Module][it.Item] = inventory.Measurement{Path: it.Path, Size: it.SizeBytes, LastWrite: it.LastWrite,
|
||||||
|
MeasuredAt: it.MeasuredAt, LastBackup: it.LastBackup, Error: it.Error, Redundancy: it.Redundancy,
|
||||||
|
Precision: it.Precision}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// declaredOn is every data item a machine's composition declares, and whether something there holds
|
||||||
|
// node-backup to measure them.
|
||||||
|
func declaredOn(plan catalogue.Resolution) ([]inventory.DeclaredData, bool) {
|
||||||
|
var out []inventory.DeclaredData
|
||||||
|
held := false
|
||||||
|
for _, m := range plan.Modules {
|
||||||
|
for _, c := range m.Claims {
|
||||||
|
if s, known := catalogue.SeatNamed(c.Name); known && s.Name == catalogue.BackupSeat {
|
||||||
|
held = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, it := range m.DataItems() {
|
||||||
|
out = append(out, inventory.DeclaredData{Module: m.Module, Item: it.ID, Class: it.Class,
|
||||||
|
Owned: it.OwnedByModule(), Protection: it.Protection()})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, held
|
||||||
|
}
|
||||||
|
|
||||||
|
// consumerCopy is one provider's account of one consumer: where, how big, and whether still active.
|
||||||
|
type consumerCopy struct {
|
||||||
|
Node, Module, Consumer string
|
||||||
|
Size *int64
|
||||||
|
Retired bool
|
||||||
|
// Class is how precious the consumer's data is: the stricter of what the provider keeps for its
|
||||||
|
// consumers and what the consumer says it keeps there (`kept-by`).
|
||||||
|
Class string
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeData is D13.
|
||||||
|
func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
if d.js == nil {
|
||||||
|
return nil, errors.New("no bus to ask the machines over")
|
||||||
|
}
|
||||||
|
open := d.open
|
||||||
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
nodes, err := open.inventory.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
heard := heardMachines(d)
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
type machine struct {
|
||||||
|
name string
|
||||||
|
declared []inventory.DeclaredData
|
||||||
|
held bool
|
||||||
|
measured map[string]map[string]inventory.Measurement
|
||||||
|
askErr error
|
||||||
|
}
|
||||||
|
var machines []*machine
|
||||||
|
for _, n := range nodes {
|
||||||
|
plan, _, err := planFor(ctx, open, n.Name)
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
// A machine that cannot be worked out declares nothing this run — which is not the same as
|
||||||
|
// declaring nothing: retiring its data on that would be acting on an unreadable result.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
declared, held := declaredOn(plan)
|
||||||
|
machines = append(machines, &machine{name: n.Name, declared: declared, held: held})
|
||||||
|
}
|
||||||
|
// Every holder asked at once, as D8 asks every ban list.
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for _, m := range machines {
|
||||||
|
if !m.held || !heard[m.name] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
wg.Add(1)
|
||||||
|
go func(m *machine) {
|
||||||
|
defer wg.Done()
|
||||||
|
asking, cancel := context.WithTimeout(ctx, dataAsk)
|
||||||
|
defer cancel()
|
||||||
|
raw, err := askSeatTool(asking, d.js.Conn(), catalogue.BackupSeat, "backed-up", m.name)
|
||||||
|
if err == nil {
|
||||||
|
m.measured, err = readHolder(raw)
|
||||||
|
}
|
||||||
|
m.askErr = err
|
||||||
|
}(m)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, m := range machines {
|
||||||
|
if m.askErr != nil {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Token: kindDataUnmeasured,
|
||||||
|
Kind: kindDataUnmeasured, Machine: m.name, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s's backup holder did not say what it measured of the data declared there, so "+
|
||||||
|
"nothing about that data is known this run: %s", m.name, firstLine(m.askErr.Error())),
|
||||||
|
Said: firstLine(m.askErr.Error())})
|
||||||
|
}
|
||||||
|
why := fmt.Sprintf("no longer declared on %s: its module was unassigned there, or is no longer pulled in", m.name)
|
||||||
|
change, err := open.inventory.RecordData(ctx, m.name, m.declared, m.measured, why, now)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("what %s holds could not be kept: %w", m.name, err)
|
||||||
|
}
|
||||||
|
for _, r := range change.Retired {
|
||||||
|
log.Printf("data: %s of %s on %s RETIRED, kept at %s: %s — `cleanup list` shows it, and only `cleanup "+
|
||||||
|
"delete` removes it (novox/hq ADR 0233)", r.Item, r.Module, r.Machine, orUnknownPath(r.Path), why)
|
||||||
|
}
|
||||||
|
for _, r := range change.Reenabled {
|
||||||
|
log.Printf("data: %s of %s on %s is declared again, no longer retired", r.Item, r.Module, r.Machine)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
records, err := open.inventory.Data(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
peaks, err := open.inventory.DataPeaks(ctx, now.Add(-shrinkWindow))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
bindings, err := open.inventory.Bindings(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
upgraded, keptBy := keptByClasses(bindings, shelf)
|
||||||
|
copies, err := consumerCopies(ctx, d.js.Conn(), open.inventory, shelf, keptBy)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, dataFindings(records, peaks, shelf, copies, upgraded, now)...)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func orUnknownPath(p string) string {
|
||||||
|
if p == "" {
|
||||||
|
return "a path its backup holder never named"
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
// consumerCopies asks every provider of a provision whose consumers' data is kept what it holds, at
|
||||||
|
// once. One that cannot answer is passed over: it says nothing about any copy, which is not a finding.
|
||||||
|
func consumerCopies(ctx context.Context, conn *nats.Conn, inv *inventory.Inventory,
|
||||||
|
shelf map[string]catalogue.Manifest, keptBy map[string]string) ([]consumerCopy, error) {
|
||||||
|
instances, err := providerInstances(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var asked []providerInstance
|
||||||
|
for _, p := range instances {
|
||||||
|
m := shelf[p.Module]
|
||||||
|
keeps := false
|
||||||
|
for provision := range m.Grants {
|
||||||
|
keeps = keeps || m.KeepsConsumerData(provision)
|
||||||
|
}
|
||||||
|
if keeps {
|
||||||
|
asked = append(asked, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
states := make([]*link.RetirementState, len(asked))
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i, p := range asked {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(i int, p providerInstance) {
|
||||||
|
defer wg.Done()
|
||||||
|
asking, cancel := context.WithTimeout(ctx, dataAsk)
|
||||||
|
defer cancel()
|
||||||
|
if s, err := askRetirement(asking, conn, p); err == nil {
|
||||||
|
states[i] = &s
|
||||||
|
}
|
||||||
|
}(i, p)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
var out []consumerCopy
|
||||||
|
for i, p := range asked {
|
||||||
|
s := states[i]
|
||||||
|
if s == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
class := consumersClass(shelf[p.Module])
|
||||||
|
for _, c := range s.Held {
|
||||||
|
cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: c,
|
||||||
|
Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+c])}
|
||||||
|
if size, ok := s.HeldSizes[c]; ok && size >= 0 {
|
||||||
|
size := size
|
||||||
|
cp.Size = &size
|
||||||
|
}
|
||||||
|
out = append(out, cp)
|
||||||
|
}
|
||||||
|
for _, r := range s.Retired {
|
||||||
|
if r.Kind != "" && r.Kind != "consumer" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: r.Consumer, Retired: true,
|
||||||
|
Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+r.Consumer])}
|
||||||
|
if r.SizeBytes != nil && *r.SizeBytes >= 0 {
|
||||||
|
cp.Size = r.SizeBytes
|
||||||
|
}
|
||||||
|
out = append(out, cp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// severityOf is how loud a finding about data of a class is: urgent for what is irreplaceable, a warning
|
||||||
|
// for anything else watched (the operator's ranking, ADR 0233).
|
||||||
|
func severityOf(class string) conditions.Severity {
|
||||||
|
if class == catalogue.ClassIrreplaceable {
|
||||||
|
return conditions.Urgent
|
||||||
|
}
|
||||||
|
return conditions.Warning
|
||||||
|
}
|
||||||
|
|
||||||
|
// consumersClass is the most precious class a provider keeps any of its consumers' data as.
|
||||||
|
func consumersClass(m catalogue.Manifest) string {
|
||||||
|
class := catalogue.ClassNone
|
||||||
|
for provision := range m.Grants {
|
||||||
|
if c, ok := m.ConsumerDataOf(provision); ok {
|
||||||
|
class = catalogue.StricterClass(class, c.Class)
|
||||||
|
} else if m.KeepsConsumerData(provision) {
|
||||||
|
class = catalogue.StricterClass(class, catalogue.ClassValuable)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return class
|
||||||
|
}
|
||||||
|
|
||||||
|
// keptByClasses is what consumers say of the data they keep with their providers (`kept-by`), read
|
||||||
|
// through where each is bound: by provider module and consumer identity, the class of that consumer's
|
||||||
|
// data there; and by provider item key (machine/module/item), the class the item holding it is held to.
|
||||||
|
func keptByClasses(bindings []inventory.Binding, shelf map[string]catalogue.Manifest) (map[string]string, map[string]string) {
|
||||||
|
upgraded, keptBy := map[string]string{}, map[string]string{}
|
||||||
|
for _, b := range bindings {
|
||||||
|
m, ok := shelf[b.Consumer]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
k, said := m.KeptByOf(b.Provision)
|
||||||
|
if !said {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
identity := catalogue.ConsumerIdentity(b.Machine, catalogue.IdentitySource(m.Slug, m.Module))
|
||||||
|
key := b.Provider.Module + "/" + identity
|
||||||
|
keptBy[key] = catalogue.StricterClass(keptBy[key], k.Class)
|
||||||
|
if pc, ok := shelf[b.Provider.Module].ConsumerDataOf(b.Provision); ok && pc.In != "" {
|
||||||
|
if _, own := shelf[b.Provider.Module].DataItem(pc.In); own {
|
||||||
|
item := b.Provider.Node + "/" + b.Provider.Module + "/" + pc.In
|
||||||
|
upgraded[item] = catalogue.StricterClass(upgraded[item], k.Class)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return upgraded, keptBy
|
||||||
|
}
|
||||||
|
|
||||||
|
// dataFindings is every condition the data on record raises now. A function of what is known, so the
|
||||||
|
// incident's shape is tested without a mesh. upgraded is the class an item is held to where a consumer
|
||||||
|
// of its module keeps data in it more precious than its own class says (`kept-by`), by its key.
|
||||||
|
func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf map[string]catalogue.Manifest,
|
||||||
|
copies []consumerCopy, upgraded map[string]string, now time.Time) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
byItem := map[string][]inventory.DataRecord{}
|
||||||
|
arrays := map[string][]inventory.DataRecord{}
|
||||||
|
type shrunk struct {
|
||||||
|
machine, dataset, class string
|
||||||
|
size, peak int64
|
||||||
|
items []string
|
||||||
|
}
|
||||||
|
shrunkDatasets := map[string]shrunk{}
|
||||||
|
for _, r := range records {
|
||||||
|
if r.DeletedAt != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
class := catalogue.StricterClass(r.Class, upgraded[r.Key()])
|
||||||
|
r.Class = class
|
||||||
|
byItem[r.Module+"/"+r.Item] = append(byItem[r.Module+"/"+r.Item], r)
|
||||||
|
item, declared := shelf[r.Module].DataItem(r.Item)
|
||||||
|
id := keyOfItem(r.Machine, r.Module, r.Item)
|
||||||
|
if r.Retired() {
|
||||||
|
if now.Sub(*r.RetiredAt) > cleanupAfter {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "cleanup",
|
||||||
|
Kind: kindCleanupWaiting, Machine: r.Machine, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||||
|
Summary: fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept at %s since %s; `cleanup "+
|
||||||
|
"delete %s %s %s --why …` once a person has decided, or assign %s there again",
|
||||||
|
r.Item, r.Module, r.Machine, r.Class, sizeWords(r.Size), int(now.Sub(*r.RetiredAt).Hours()/24),
|
||||||
|
orUnknownPath(r.Path), r.RetiredWhy, r.Machine, r.Module, r.Item, r.Module)})
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !catalogue.Watched(class) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
severity := severityOf(class)
|
||||||
|
if r.Redundancy != nil {
|
||||||
|
where := r.Machine + "/" + r.Redundancy.Kind + ":" + r.Redundancy.Where
|
||||||
|
arrays[where] = append(arrays[where], r)
|
||||||
|
} else if declared && item.Redundancy != "" && r.MeasuredAt != nil && r.MeasureError == "" {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindProtectionMissing,
|
||||||
|
Kind: kindProtectionMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
|
||||||
|
Summary: fmt.Sprintf("%s of %s on %s (%s) is said to be protected by the redundancy of the storage it is on, "+
|
||||||
|
"and %s is on nothing the backup holder can read as redundant: it has no protection the mesh can see",
|
||||||
|
r.Item, r.Module, r.Machine, class, orUnknownPath(r.Path))})
|
||||||
|
}
|
||||||
|
if r.MeasureError != "" && strings.Contains(r.MeasureError, "does not exist") {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataMissing,
|
||||||
|
Kind: kindDataMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
|
||||||
|
Summary: fmt.Sprintf("%s of %s on %s (%s) is gone: %s does not exist any more", r.Item, r.Module, r.Machine,
|
||||||
|
class, orUnknownPath(r.Path))})
|
||||||
|
} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) &&
|
||||||
|
*r.Size*2 < peak && peak-*r.Size >= shrinkFloor && inventory.Dataset(r.Precision) != "" {
|
||||||
|
// Several items on one dataset share its size: one condition for the dataset, as loud as the
|
||||||
|
// most precious item on it.
|
||||||
|
k := r.Machine + "/" + inventory.Dataset(r.Precision)
|
||||||
|
ds := shrunkDatasets[k]
|
||||||
|
ds.machine, ds.dataset, ds.size, ds.peak = r.Machine, inventory.Dataset(r.Precision), *r.Size, peak
|
||||||
|
ds.class = catalogue.StricterClass(ds.class, class)
|
||||||
|
ds.items = append(ds.items, r.Module+"/"+r.Item)
|
||||||
|
shrunkDatasets[k] = ds
|
||||||
|
} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) &&
|
||||||
|
*r.Size*2 < peak && peak-*r.Size >= shrinkFloor {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataShrank,
|
||||||
|
Kind: kindDataShrank, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
|
||||||
|
Summary: fmt.Sprintf("%s of %s on %s (%s) shrank to %s from %s within %d days — more than half of what it "+
|
||||||
|
"held is gone. If that was meant, silence this with why; if not, `node-backup.restore` puts the last "+
|
||||||
|
"good copy beside it", r.Item, r.Module, r.Machine, class, sizeWords(r.Size), sizeWords(&peak),
|
||||||
|
int(shrinkWindow.Hours()/24))})
|
||||||
|
}
|
||||||
|
if !declared {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if within := item.ActiveWithin(); within > 0 && r.LastWrite != nil && now.Sub(*r.LastWrite) > within {
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataQuiet,
|
||||||
|
Kind: kindDataQuiet, Machine: r.Machine, Severity: severity,
|
||||||
|
Summary: fmt.Sprintf("%s of %s on %s is written all the time, and has not been since %s (its bound is %s): "+
|
||||||
|
"whatever writes it has stopped", r.Item, r.Module, r.Machine, r.LastWrite.UTC().Format(time.RFC3339),
|
||||||
|
within)})
|
||||||
|
}
|
||||||
|
// A backup is required of what is irreplaceable and copied; of what is valuable it is the standard
|
||||||
|
// plan, said only where the machine was measured — where a holder is there to take it.
|
||||||
|
if item.BackedUp() && (class == catalogue.ClassIrreplaceable || r.MeasuredAt != nil) {
|
||||||
|
within := item.BackupWithin()
|
||||||
|
switch {
|
||||||
|
case r.LastBackup == nil && now.Sub(r.FirstSeen) > within:
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale,
|
||||||
|
Kind: kindBackupStale, Machine: r.Machine, Severity: severity,
|
||||||
|
Summary: fmt.Sprintf("%s of %s on %s is %s and has no good backup on record, %s after it was first "+
|
||||||
|
"declared — is node-backup held there, and do its nights succeed? (`node-backup.backed-up`)",
|
||||||
|
r.Item, r.Module, r.Machine, class, now.Sub(r.FirstSeen).Round(time.Hour))})
|
||||||
|
case r.LastBackup != nil && now.Sub(*r.LastBackup) > within:
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale,
|
||||||
|
Kind: kindBackupStale, Machine: r.Machine, Severity: severity,
|
||||||
|
Summary: fmt.Sprintf("%s of %s on %s is %s and its newest good backup is from %s, older than its bound "+
|
||||||
|
"of %s", r.Item, r.Module, r.Machine, class, r.LastBackup.UTC().Format(time.RFC3339), within)})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, k := range keysSorted(shrunkDatasets) {
|
||||||
|
ds := shrunkDatasets[k]
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
|
||||||
|
ID: ds.machine + ".dataset." + strings.ReplaceAll(ds.dataset, "/", "-"), Token: kindDataShrank,
|
||||||
|
Kind: kindDataShrank, Machine: ds.machine, Severity: severityOf(ds.class), Resolver: conditions.ResolverOperator,
|
||||||
|
Summary: fmt.Sprintf("the dataset %s on %s shrank to %s from %s within %d days — more than half of what it held "+
|
||||||
|
"is gone; it holds %s", ds.dataset, ds.machine, sizeWords(&ds.size), sizeWords(&ds.peak),
|
||||||
|
int(shrinkWindow.Hours()/24), strings.Join(ds.items, ", "))})
|
||||||
|
}
|
||||||
|
// The redundant storage watched data is on: one condition per array, as loud as the most precious
|
||||||
|
// item on it — the array, not each item, is what degrades.
|
||||||
|
for _, where := range keysSorted(arrays) {
|
||||||
|
rs := arrays[where]
|
||||||
|
red := rs[0].Redundancy
|
||||||
|
if red.Healthy != nil && *red.Healthy {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
class, machine := catalogue.ClassValuable, rs[0].Machine
|
||||||
|
var names []string
|
||||||
|
for _, r := range rs {
|
||||||
|
class = catalogue.StricterClass(class, r.Class)
|
||||||
|
names = append(names, r.Module+"/"+r.Item)
|
||||||
|
}
|
||||||
|
state := "could not be read"
|
||||||
|
if red.Healthy != nil {
|
||||||
|
state = "is NOT healthy"
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
|
||||||
|
ID: machine + ".array." + strings.NewReplacer("/", "-", ":", "-").Replace(red.Kind+"-"+red.Where),
|
||||||
|
Token: kindArrayDegraded, Kind: kindArrayDegraded, Machine: machine, Severity: severityOf(class),
|
||||||
|
Resolver: conditions.ResolverOperator,
|
||||||
|
Summary: fmt.Sprintf("the %s storage %s on %s %s: %s — and it is what protects %s", red.Kind, red.Where, machine,
|
||||||
|
state, firstLine(red.Said), strings.Join(names, ", "))})
|
||||||
|
}
|
||||||
|
// The same item on several machines: a copy that is in use and far smaller than one kept elsewhere is
|
||||||
|
// an empty replacement. Only against a retired copy — a module running on two machines on purpose
|
||||||
|
// keeps two different sets of data.
|
||||||
|
for _, key := range keysSorted(byItem) {
|
||||||
|
rs := byItem[key]
|
||||||
|
for _, a := range rs {
|
||||||
|
if a.Retired() || a.Size == nil || !catalogue.Watched(a.Class) || !inventory.Comparable(a.Precision) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, o := range rs {
|
||||||
|
if o.Machine == a.Machine || !o.Retired() || o.Size == nil || !inventory.Comparable(o.Precision) ||
|
||||||
|
!replacedByLess(*a.Size, *o.Size) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
|
||||||
|
ID: keyOfItem(a.Machine, a.Module, a.Item), Token: kindEmptyReplacement, Kind: kindEmptyReplacement,
|
||||||
|
Machine: a.Machine, Also: []string{o.Machine}, Severity: severityOf(a.Class), Resolver: conditions.ResolverOperator,
|
||||||
|
Summary: fmt.Sprintf("%s of %s on %s holds %s, and the copy %s kept on %s holds %s: %s is running on "+
|
||||||
|
"an empty replacement of its data. Move the data, or assign it back where its data is",
|
||||||
|
a.Item, a.Module, a.Machine, sizeWords(a.Size), o.Module, o.Machine, sizeWords(o.Size), a.Module)})
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out = append(out, consumerFindings(copies)...)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// replacedByLess is whether a copy in use is an empty replacement of a copy kept elsewhere: less than
|
||||||
|
// half of it, and at least shrinkFloor less.
|
||||||
|
func replacedByLess(inUse, kept int64) bool {
|
||||||
|
return inUse*2 < kept && kept-inUse >= shrinkFloor
|
||||||
|
}
|
||||||
|
|
||||||
|
// consumerFindings is the same question of consumers' data at providers: one consumer, the same
|
||||||
|
// provider module on two machines.
|
||||||
|
func consumerFindings(copies []consumerCopy) []conditions.Observation {
|
||||||
|
by := map[string][]consumerCopy{}
|
||||||
|
for _, c := range copies {
|
||||||
|
k := c.Module + "/" + c.Consumer
|
||||||
|
by[k] = append(by[k], c)
|
||||||
|
}
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, k := range keysSorted(by) {
|
||||||
|
cs := by[k]
|
||||||
|
if len(cs) < 2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
for _, a := range cs {
|
||||||
|
if a.Retired || a.Size == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, o := range cs {
|
||||||
|
if o.Node == a.Node || o.Size == nil || !replacedByLess(*a.Size, *o.Size) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
state := "active"
|
||||||
|
if o.Retired {
|
||||||
|
state = "retired"
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider,
|
||||||
|
ID: a.Module + "." + a.Node + "." + a.Consumer, Token: kindEmptyReplacement, Kind: kindEmptyReplacement,
|
||||||
|
Machine: a.Node, Also: []string{o.Node}, Severity: severityOf(catalogue.StricterClass(a.Class, o.Class)),
|
||||||
|
Resolver: conditions.ResolverOperator,
|
||||||
|
Summary: fmt.Sprintf("%s's data at %s on %s holds %s, and its %s copy at %s on %s holds %s: the "+
|
||||||
|
"consumer is using an empty replacement of its data (issue 273's shape). Pin it back to %s, or move "+
|
||||||
|
"the data first", a.Consumer, a.Module, a.Node, sizeWords(a.Size), state, o.Module, o.Node,
|
||||||
|
sizeWords(o.Size), o.Node)})
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if found {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if found {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var active []consumerCopy
|
||||||
|
for _, c := range cs {
|
||||||
|
if !c.Retired {
|
||||||
|
active = append(active, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(active) >= 2 {
|
||||||
|
var where []string
|
||||||
|
var also []string
|
||||||
|
for _, c := range active {
|
||||||
|
where = append(where, c.Node+" ("+sizeWords(c.Size)+")")
|
||||||
|
also = append(also, c.Node)
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider,
|
||||||
|
ID: active[0].Module + "." + active[0].Consumer, Token: kindDataHeldTwice, Kind: kindDataHeldTwice,
|
||||||
|
Machine: active[0].Node, Also: also[1:], Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||||
|
Summary: fmt.Sprintf("%s has active data at %s on %d machines — %s — and only one is the one it uses",
|
||||||
|
active[0].Consumer, active[0].Module, len(active), strings.Join(where, ", "))})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func keysSorted[V any](m map[string]V) []string {
|
||||||
|
out := make([]string, 0, len(m))
|
||||||
|
for k := range m {
|
||||||
|
out = append(out, k)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the `data` verb ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const dataUsage = "data [--json] [--machine <name>] [--retired]"
|
||||||
|
|
||||||
|
// dataRow is one item as `data` lists it.
|
||||||
|
type dataRow struct {
|
||||||
|
Machine string `json:"machine"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
Item string `json:"item"`
|
||||||
|
Class string `json:"class"`
|
||||||
|
Path string `json:"path,omitempty"`
|
||||||
|
Protection string `json:"protection,omitempty"`
|
||||||
|
// Array is the redundant storage it is on and its state, where its holder could tell.
|
||||||
|
Array string `json:"array,omitempty"`
|
||||||
|
Unmeasured string `json:"unmeasured,omitempty"`
|
||||||
|
// Precision says what the size is: exact, a dataset's whole size, partial, or none.
|
||||||
|
Precision string `json:"precision,omitempty"`
|
||||||
|
SizeBytes *int64 `json:"size-bytes,omitempty"`
|
||||||
|
LastWrite string `json:"last-write,omitempty"`
|
||||||
|
MeasuredAt string `json:"measured-at,omitempty"`
|
||||||
|
LastBackup string `json:"last-backup,omitempty"`
|
||||||
|
BackupDue string `json:"backup-within,omitempty"`
|
||||||
|
Retired string `json:"retired,omitempty"`
|
||||||
|
RetiredWhy string `json:"retired-why,omitempty"`
|
||||||
|
Deleted string `json:"deleted,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// dataCommand is `data`: every item every machine declares, or held retired, as the self-check last
|
||||||
|
// found it.
|
||||||
|
func dataCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("data", flag.ContinueOnError)
|
||||||
|
asJSON := set.Bool("json", false, "as data")
|
||||||
|
only := set.String("machine", "", "one machine")
|
||||||
|
retiredOnly := set.Bool("retired", false, "only what is retired")
|
||||||
|
if rest, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
} else if len(rest) > 0 {
|
||||||
|
return errors.New(dataUsage)
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
records, err := open.inventory.Data(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rows := dataRows(records, shelf, *only, *retiredOnly)
|
||||||
|
if *asJSON {
|
||||||
|
return printJSON(map[string]any{"data": rows})
|
||||||
|
}
|
||||||
|
if len(rows) == 0 {
|
||||||
|
fmt.Println("no data on record: the self-check (D13) records what each machine declares on its next run")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, r := range rows {
|
||||||
|
state := ""
|
||||||
|
switch {
|
||||||
|
case r.Deleted != "":
|
||||||
|
state = " DELETED " + r.Deleted
|
||||||
|
case r.Retired != "":
|
||||||
|
state = " RETIRED " + r.Retired + " — " + r.RetiredWhy
|
||||||
|
}
|
||||||
|
fmt.Printf("%s %s/%s %s %s %s protected by %s%s\n", r.Machine, r.Module, r.Item, r.Class,
|
||||||
|
sizeWords(r.SizeBytes), orUnknownPath(r.Path), orNothingWord(r.Protection), state)
|
||||||
|
if r.Array != "" {
|
||||||
|
fmt.Printf(" on %s\n", r.Array)
|
||||||
|
}
|
||||||
|
if r.Precision != "" && r.Precision != "exact" {
|
||||||
|
fmt.Printf(" size: %s\n", r.Precision)
|
||||||
|
}
|
||||||
|
if r.Unmeasured != "" {
|
||||||
|
fmt.Printf(" not measured: %s\n", r.Unmeasured)
|
||||||
|
}
|
||||||
|
if r.Class == catalogue.ClassCache {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fmt.Printf(" last write %s, measured %s, last backup %s%s\n", orNever(r.LastWrite), orNever(r.MeasuredAt),
|
||||||
|
orNever(r.LastBackup), within(r.BackupDue))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func dataRows(records []inventory.DataRecord, shelf map[string]catalogue.Manifest, only string, retiredOnly bool) []dataRow {
|
||||||
|
rows := []dataRow{}
|
||||||
|
stamp := func(t *time.Time) string {
|
||||||
|
if t == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return t.UTC().Format(time.RFC3339)
|
||||||
|
}
|
||||||
|
for _, r := range records {
|
||||||
|
if only != "" && r.Machine != only {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if retiredOnly && !r.Retired() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
row := dataRow{Machine: r.Machine, Module: r.Module, Item: r.Item, Class: r.Class, Path: r.Path,
|
||||||
|
Protection: r.Protection, Unmeasured: r.MeasureError, Precision: r.Precision, SizeBytes: r.Size, LastWrite: stamp(r.LastWrite), MeasuredAt: stamp(r.MeasuredAt),
|
||||||
|
LastBackup: stamp(r.LastBackup), Retired: stamp(r.RetiredAt), RetiredWhy: r.RetiredWhy,
|
||||||
|
Deleted: stamp(r.DeletedAt)}
|
||||||
|
if it, ok := shelf[r.Module].DataItem(r.Item); ok && it.BackedUp() {
|
||||||
|
row.BackupDue = it.BackupWithin().String()
|
||||||
|
}
|
||||||
|
if red := r.Redundancy; red != nil {
|
||||||
|
state := "state unread"
|
||||||
|
if red.Healthy != nil && *red.Healthy {
|
||||||
|
state = "healthy"
|
||||||
|
} else if red.Healthy != nil {
|
||||||
|
state = "NOT HEALTHY"
|
||||||
|
}
|
||||||
|
row.Array = red.Kind + " " + red.Where + ", " + state
|
||||||
|
}
|
||||||
|
rows = append(rows, row)
|
||||||
|
}
|
||||||
|
return rows
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNothingWord(s string) string {
|
||||||
|
if s == "" || s == "none" {
|
||||||
|
return "nothing"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNever(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "never"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func within(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return " (not backed up)"
|
||||||
|
}
|
||||||
|
return " (bound " + s + ")"
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- cleanup of retired own data ---------------------------------------------------------------
|
||||||
|
|
||||||
|
// The tools a node-backup holder serves to delete one retired item (novox/hq ADR 0233): the first
|
||||||
|
// takes a last restore point of it, tagged as retired, and only then removes it — in the background,
|
||||||
|
// because a large item outlasts any call — and the second says how that went. Module tools, not seat
|
||||||
|
// verbs: only the controller's `cleanup delete` calls them, as it calls a provider's provisioner_delete.
|
||||||
|
const (
|
||||||
|
ToolDeleteRetired = "backup_delete_retired"
|
||||||
|
ToolDeletedOutcome = "backup_deleted"
|
||||||
|
)
|
||||||
|
|
||||||
|
// deletionWait is how long `cleanup delete` follows a deletion before handing it back to the person.
|
||||||
|
var deletionWait = 8 * time.Minute
|
||||||
|
|
||||||
|
// deletionPoll is how often it asks.
|
||||||
|
var deletionPoll = 5 * time.Second
|
||||||
|
|
||||||
|
// deletion is a holder's account of one deletion.
|
||||||
|
type deletion struct {
|
||||||
|
Started bool `json:"started"`
|
||||||
|
Running bool `json:"running"`
|
||||||
|
Done bool `json:"done"`
|
||||||
|
OK bool `json:"ok"`
|
||||||
|
Snapshot string `json:"snapshot"`
|
||||||
|
Error string `json:"error"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// retiredData is every retired item on record, as `cleanup list` shows them.
|
||||||
|
func retiredData(records []inventory.DataRecord, now time.Time) []retiredRow {
|
||||||
|
var out []retiredRow
|
||||||
|
for _, r := range records {
|
||||||
|
if !r.Retired() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, retiredRow{Node: r.Machine, Module: r.Module, Consumer: r.Item, Kind: retiredDataKind,
|
||||||
|
RetiredAt: r.RetiredAt.UTC().Format(time.RFC3339), AgeDays: int(now.Sub(*r.RetiredAt).Hours() / 24),
|
||||||
|
SizeBytes: r.Size, Why: r.RetiredWhy, Path: r.Path, Class: r.Class})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// retiredDataKind is what `cleanup list` calls a module's own retired data, beside a provider's consumer.
|
||||||
|
const retiredDataKind = "own-data"
|
||||||
|
|
||||||
|
// holderOn is the module holding node-backup on a machine.
|
||||||
|
func holderOn(ctx context.Context, inv *inventory.Inventory, machine string) (string, error) {
|
||||||
|
held, err := inv.Holdings(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
for _, h := range held {
|
||||||
|
if s, known := catalogue.SeatNamed(h.Claim); known && s.Name == catalogue.BackupSeat && h.Node == machine {
|
||||||
|
return h.Module, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("nothing holds %s on %s, and it is the backup holder that deletes retired data there "+
|
||||||
|
"(after a last restore point)", catalogue.BackupSeat, machine)
|
||||||
|
}
|
||||||
|
|
||||||
|
// deleteRetiredData has a machine's backup holder delete one retired item: never one declared now, and
|
||||||
|
// never one not retired. The holder takes a last restore point of it first, so the deletion can be
|
||||||
|
// undone until a person forgets that restore point; the record says deleted only once the holder says
|
||||||
|
// it is.
|
||||||
|
func deleteRetiredData(ctx context.Context, conn *nats.Conn, open *stores, r inventory.DataRecord, f handActFlags) error {
|
||||||
|
inv := open.inventory
|
||||||
|
if !r.Retired() {
|
||||||
|
return fmt.Errorf("%s of %s on %s is not retired — only retired data is deleted. Nothing was done",
|
||||||
|
r.Item, r.Module, r.Machine)
|
||||||
|
}
|
||||||
|
if r.Path == "" {
|
||||||
|
return fmt.Errorf("%s of %s on %s was never measured, so where it is was never said; nothing was deleted",
|
||||||
|
r.Item, r.Module, r.Machine)
|
||||||
|
}
|
||||||
|
if plan, _, err := planFor(ctx, open, r.Machine); err == nil {
|
||||||
|
for _, m := range plan.Modules {
|
||||||
|
if _, still := m.DataItem(r.Item); still && m.Module == r.Module {
|
||||||
|
return fmt.Errorf("%s runs on %s again and declares %s: it is not retired any more. Nothing was done",
|
||||||
|
r.Module, r.Machine, r.Item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
holder, err := holderOn(ctx, inv, r.Machine)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
f.record(ctx, "cleanup delete", []string{r.Machine, r.Module, r.Item})
|
||||||
|
args := map[string]any{"module": r.Module, "item": r.Item, "path": r.Path, "confirm": r.Item,
|
||||||
|
"why": strings.TrimSpace(*f.why), "by": link.Caller(), "via": link.ViaController}
|
||||||
|
ask := func(tool string) (deletion, error) {
|
||||||
|
var d deletion
|
||||||
|
answer, err := link.AskModuleToolOn(ctx, conn, holder, tool, r.Machine, args, 25*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
return d, err
|
||||||
|
}
|
||||||
|
if answer.Error != "" {
|
||||||
|
return d, fmt.Errorf("%s on %s refused: %s", holder, r.Machine, answer.Error)
|
||||||
|
}
|
||||||
|
return d, unmarshalAnswer(answer, &d)
|
||||||
|
}
|
||||||
|
d, err := ask(ToolDeleteRetired)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for waited := time.Duration(0); !d.Done && waited < deletionWait; waited += deletionPoll {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-time.After(deletionPoll):
|
||||||
|
}
|
||||||
|
if d, err = ask(ToolDeletedOutcome); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case !d.Done:
|
||||||
|
fmt.Printf("%s on %s is still taking the last restore point of %s and deleting it; `cleanup list` keeps "+
|
||||||
|
"showing it until the holder says it is done — the same `cleanup delete` again reads how it went\n",
|
||||||
|
holder, r.Machine, r.Path)
|
||||||
|
return nil
|
||||||
|
case !d.OK:
|
||||||
|
return fmt.Errorf("%s on %s did NOT delete %s: %s", holder, r.Machine, r.Path, d.Error)
|
||||||
|
}
|
||||||
|
if err := inv.MarkDataDeleted(ctx, r.Machine, r.Module, r.Item, link.Caller(), strings.TrimSpace(*f.why), time.Now()); err != nil {
|
||||||
|
return fmt.Errorf("%s deleted %s on %s, and it could not be recorded: %w", holder, r.Path, r.Machine, err)
|
||||||
|
}
|
||||||
|
fmt.Printf("%s on %s deleted %s of %s (%s, %s); its last restore point is %s, kept until a person forgets it\n",
|
||||||
|
holder, r.Machine, r.Item, r.Module, r.Path, sizeWords(r.Size), orNever(d.Snapshot))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// keptOnUnassign says, for an unassignment, the irreplaceable and valuable data each module leaves in its
|
||||||
|
// own directories on the machine:
|
||||||
|
// kept, and retired at the self-check's next run.
|
||||||
|
func keptOnUnassign(ctx context.Context, inv *inventory.Inventory, machine string, modules []string) []string {
|
||||||
|
records, err := inv.Data(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return []string{"what it leaves behind could not be read from the mesh's record: " + err.Error()}
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, r := range records {
|
||||||
|
if r.Machine != machine || r.DeletedAt != nil || !catalogue.Retires(r.Class) || !r.Owned {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, m := range modules {
|
||||||
|
if r.Module == m {
|
||||||
|
out = append(out, fmt.Sprintf("%s's %s (%s, %s) stays where it is: it is %s, so it is retired, "+
|
||||||
|
"never removed — `cleanup list` shows it, `cleanup delete` alone removes it (novox/hq ADR 0233)",
|
||||||
|
r.Module, r.Item, orUnknownPath(r.Path), sizeWords(r.Size), r.Class))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,460 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
func bytesOf(n int64) *int64 { return &n }
|
||||||
|
|
||||||
|
func when(t time.Time) *time.Time { return &t }
|
||||||
|
|
||||||
|
func shelfFor(t *testing.T, manifests ...string) map[string]catalogue.Manifest {
|
||||||
|
t.Helper()
|
||||||
|
out := map[string]catalogue.Manifest{}
|
||||||
|
for _, raw := range manifests {
|
||||||
|
m, err := catalogue.ParseManifest([]byte(raw))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out[m.Module] = m
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
const houseManifest = `{"module":"house","version":"1",
|
||||||
|
"data":{"own":[{"id":"config","path":"${dir:config}","class":"irreplaceable","active":"1d"}]},
|
||||||
|
"resources":[{"id":"config","type":"directory","mode":"0700"}]}`
|
||||||
|
|
||||||
|
func findingsByKind(obs []conditions.Observation) map[string]conditions.Observation {
|
||||||
|
out := map[string]conditions.Observation{}
|
||||||
|
for _, o := range obs {
|
||||||
|
out[o.Kind] = o
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// THE INCIDENT (issue 273), replayed against what D13 reads: five applications on the home server bound,
|
||||||
|
// by one changed rule, to the store on the control node, which made each an empty database — while
|
||||||
|
// their real databases, hundreds of megabytes each, sat on the home server's own store, by then retired
|
||||||
|
// because the mesh no longer asked for them there. Each is an empty replacement, naming both machines
|
||||||
|
// and the pin back: a warning for the store's consumers, whose data is valuable; urgent for one that says
|
||||||
|
// its data there is irreplaceable (`kept-by`).
|
||||||
|
func TestAnEmptyReplacementOfAConsumersDataIsSaid(t *testing.T) {
|
||||||
|
var copies []consumerCopy
|
||||||
|
for _, app := range []string{"mesh_home_board", "mesh_home_flows", "mesh_home_agents", "mesh_home_game", "mesh_home_cars"} {
|
||||||
|
copies = append(copies,
|
||||||
|
consumerCopy{Node: "home", Module: "postgres", Consumer: app, Size: bytesOf(400 << 20), Retired: true, Class: "valuable"},
|
||||||
|
consumerCopy{Node: "anchor", Module: "postgres", Consumer: app, Size: bytesOf(9 << 20), Class: "valuable"})
|
||||||
|
}
|
||||||
|
copies[1].Class = "irreplaceable" // the photo site's own database says so
|
||||||
|
got := dataFindings(nil, nil, nil, copies, nil, time.Now())
|
||||||
|
if len(got) != 5 {
|
||||||
|
t.Fatalf("%d findings for five empty replacements: %+v", len(got), got)
|
||||||
|
}
|
||||||
|
for i, o := range got {
|
||||||
|
want := conditions.Warning
|
||||||
|
if strings.Contains(o.ID, "mesh_home_board") {
|
||||||
|
want = conditions.Urgent
|
||||||
|
}
|
||||||
|
_ = i
|
||||||
|
if o.Kind != kindEmptyReplacement || o.Severity != want || o.Machine != "anchor" ||
|
||||||
|
len(o.Also) != 1 || o.Also[0] != "home" || !strings.Contains(o.Summary, "Pin it back to home") {
|
||||||
|
t.Errorf("%+v", o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// While the old copy is still active (the first ten minutes), it is the same finding.
|
||||||
|
copies[0].Retired = false
|
||||||
|
copies[1].Class = "valuable"
|
||||||
|
if got := dataFindings(nil, nil, nil, copies[:2], nil, time.Now()); len(got) != 1 || got[0].Kind != kindEmptyReplacement {
|
||||||
|
t.Fatalf("with the old copy still active: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A move a person made — the data moved first, then pinned — leaves a full copy at the new provider and
|
||||||
|
// a retired one at the old: nothing to say here; `cleanup` covers the old one.
|
||||||
|
func TestADeliberateMoveIsNoEmptyReplacement(t *testing.T) {
|
||||||
|
copies := []consumerCopy{
|
||||||
|
{Node: "home", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(400 << 20), Retired: true},
|
||||||
|
{Node: "anchor", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(402 << 20)},
|
||||||
|
}
|
||||||
|
if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 {
|
||||||
|
t.Fatalf("a deliberate move raised %+v", got)
|
||||||
|
}
|
||||||
|
// Two small databases differing by less than the floor are not a finding either.
|
||||||
|
copies[0].Size, copies[1].Size = bytesOf(12<<20), bytesOf(8<<20)
|
||||||
|
if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 {
|
||||||
|
t.Fatalf("noise between two empty databases raised %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Where a provider cannot say sizes, a consumer active at two providers is still said — as a warning,
|
||||||
|
// since which one is empty cannot be told.
|
||||||
|
func TestConsumerDataActiveTwiceWithoutSizesIsAWarning(t *testing.T) {
|
||||||
|
copies := []consumerCopy{
|
||||||
|
{Node: "home", Module: "minio", Consumer: "mesh_home_photos"},
|
||||||
|
{Node: "anchor", Module: "minio", Consumer: "mesh_home_photos"},
|
||||||
|
}
|
||||||
|
got := dataFindings(nil, nil, nil, copies, nil, time.Now())
|
||||||
|
if len(got) != 1 || got[0].Kind != kindDataHeldTwice || got[0].Severity != conditions.Warning {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The same incident for a module's own data: a module unassigned from one machine and assigned on
|
||||||
|
// another starts over in an empty directory while its full one is kept, retired, where it was.
|
||||||
|
func TestAnEmptyReplacementOfAModulesOwnDataIsUrgent(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
retired := now.Add(-time.Hour)
|
||||||
|
records := []inventory.DataRecord{
|
||||||
|
{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config",
|
||||||
|
Size: bytesOf(2 << 30), RetiredAt: &retired, FirstSeen: now.Add(-90 * 24 * time.Hour)},
|
||||||
|
{Machine: "anchor", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config",
|
||||||
|
Size: bytesOf(1 << 20), FirstSeen: now.Add(-time.Hour), LastWrite: when(now)},
|
||||||
|
}
|
||||||
|
got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))
|
||||||
|
o, ok := got[kindEmptyReplacement]
|
||||||
|
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "home" {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
// The same of a valuable item is a warning.
|
||||||
|
records[0].Class, records[1].Class = "valuable", "valuable"
|
||||||
|
if o := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))[kindEmptyReplacement]; o.Severity != conditions.Warning {
|
||||||
|
t.Fatalf("a valuable empty replacement: %+v", o)
|
||||||
|
}
|
||||||
|
records[0].Class, records[1].Class = "irreplaceable", "irreplaceable"
|
||||||
|
// Two machines running a module on purpose, both active, keep two sets of data: nothing to say.
|
||||||
|
records[0].RetiredAt = nil
|
||||||
|
if got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now)); got[kindEmptyReplacement].Kind != "" {
|
||||||
|
t.Fatalf("two active copies were read as a replacement: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An irreplaceable item that lost more than half of its largest size in a week is urgent; a smaller loss,
|
||||||
|
// or a loss under the floor, is not a finding.
|
||||||
|
func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
|
||||||
|
Size: bytesOf(300 << 20), FirstSeen: now.Add(-30 * 24 * time.Hour), LastWrite: when(now), LastBackup: when(now)}
|
||||||
|
shelf := shelfFor(t, houseManifest)
|
||||||
|
o := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): 1 << 30}, shelf, nil, nil, now))[kindDataShrank]
|
||||||
|
if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "shrank") {
|
||||||
|
t.Fatalf("%+v", o)
|
||||||
|
}
|
||||||
|
for _, peak := range []int64{500 << 20, 20 << 20} {
|
||||||
|
if got := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): peak}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" {
|
||||||
|
t.Errorf("a peak of %d raised a shrink", peak)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
small := r
|
||||||
|
small.Size = bytesOf(1 << 20)
|
||||||
|
if got := findingsByKind(dataFindings([]inventory.DataRecord{small}, map[string]int64{r.Key(): 10 << 20}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" {
|
||||||
|
t.Error("a loss under the floor raised a shrink")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Data said to be written all the time and not written; data with no backup or an old one — urgent when
|
||||||
|
// irreplaceable, a warning when valuable; and a new item given its bound before it is said.
|
||||||
|
func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
shelf := shelfFor(t, houseManifest)
|
||||||
|
r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
|
||||||
|
Size: bytesOf(1 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), LastWrite: when(now.Add(-3 * 24 * time.Hour)),
|
||||||
|
LastBackup: when(now.Add(-72 * time.Hour))}
|
||||||
|
got := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now))
|
||||||
|
if got[kindDataQuiet].Severity != conditions.Urgent || got[kindBackupStale].Severity != conditions.Urgent {
|
||||||
|
t.Fatalf("irreplaceable: %+v", got)
|
||||||
|
}
|
||||||
|
valuable := r
|
||||||
|
valuable.Class, valuable.MeasuredAt = "valuable", when(now) // measured: a holder is there to take its backup
|
||||||
|
if got := findingsByKind(dataFindings([]inventory.DataRecord{valuable}, nil, shelf, nil, nil, now)); got[kindDataQuiet].Severity != conditions.Warning ||
|
||||||
|
got[kindBackupStale].Severity != conditions.Warning {
|
||||||
|
t.Fatalf("valuable: %+v", got)
|
||||||
|
}
|
||||||
|
never := r
|
||||||
|
never.LastBackup = nil
|
||||||
|
if o := findingsByKind(dataFindings([]inventory.DataRecord{never}, nil, shelf, nil, nil, now))[kindBackupStale]; !strings.Contains(o.Summary, "no good backup") {
|
||||||
|
t.Fatalf("never backed up: %+v", o)
|
||||||
|
}
|
||||||
|
fresh := never
|
||||||
|
fresh.FirstSeen, fresh.LastWrite = now.Add(-time.Hour), when(now)
|
||||||
|
if got := dataFindings([]inventory.DataRecord{fresh}, nil, shelf, nil, nil, now); len(got) != 0 {
|
||||||
|
t.Fatalf("an item declared an hour ago, before its first night, raised %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An item retired more than thirty days waits for a person; less, it is only listed.
|
||||||
|
func TestRetiredDataWaitingThirtyDaysIsSaid(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
old, recent := now.Add(-31*24*time.Hour), now.Add(-2*24*time.Hour)
|
||||||
|
records := []inventory.DataRecord{
|
||||||
|
{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &old},
|
||||||
|
{Machine: "home", Module: "attic", Item: "boxes", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &recent},
|
||||||
|
}
|
||||||
|
got := dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now)
|
||||||
|
if len(got) != 1 || got[0].Kind != kindCleanupWaiting || !strings.Contains(got[0].Summary, "cleanup delete home house config") {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
if rows := retiredData(records, now); len(rows) != 2 || rows[0].Kind != retiredDataKind {
|
||||||
|
t.Fatalf("cleanup list: %+v", rows)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The holder's answer reads into measurements, by module and item.
|
||||||
|
func TestTheHoldersAnswerIsRead(t *testing.T) {
|
||||||
|
raw := []byte(`[{"module":"postgres","runs":1,"paths":["/var/lib/mesh-store/dumps"],"lastNight":null,"restorePoints":3,
|
||||||
|
"data":[{"item":"store","class":"irreplaceable","path":"/var/lib/mesh-store","covered_by":"/var/lib/mesh-store/dumps",
|
||||||
|
"size_bytes":1073741824,"last_write":"2026-10-06T10:00:00Z","measured_at":"2026-10-06T10:05:00Z","last_backup":"2026-10-06T03:10:00Z"}]}]`)
|
||||||
|
got, err := readHolder(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := got["postgres"]["store"]
|
||||||
|
if m.Path != "/var/lib/mesh-store" || m.Size == nil || *m.Size != 1<<30 || m.LastBackup == nil || m.MeasuredAt == nil {
|
||||||
|
t.Fatalf("%+v", m)
|
||||||
|
}
|
||||||
|
// An older holder, which says no data, reads as nothing measured rather than a failure.
|
||||||
|
if got, err := readHolder([]byte(`[{"module":"postgres","runs":1,"paths":[]}]`)); err != nil || len(got) != 0 {
|
||||||
|
t.Fatalf("%v, %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakeHolder answers node-backup's `backed-up` on one machine over a real bus, with what it is told it
|
||||||
|
// measured.
|
||||||
|
type fakeHolder struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
modules []map[string]any
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeHolder) set(modules ...map[string]any) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
f.modules = modules
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeHolder) serve(t *testing.T, conn *nats.Conn, node string) {
|
||||||
|
t.Helper()
|
||||||
|
sub, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
body, _ := json.Marshal(map[string]any{"result": f.modules, "error": "", "node": node})
|
||||||
|
_ = m.Respond(body)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||||
|
if err := conn.Flush(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func measuredHouse(path string, size int64, at time.Time) map[string]any {
|
||||||
|
return map[string]any{"module": "house", "runs": 0, "paths": []string{path}, "data": []map[string]any{{
|
||||||
|
"item": "config", "class": "irreplaceable", "path": path, "covered_by": path, "size_bytes": size,
|
||||||
|
"last_write": at, "measured_at": at, "last_backup": at}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// UNASSIGNING A MODULE WITH IRREPLACEABLE DATA KEEPS THE DATA, and assigning it elsewhere onto an empty
|
||||||
|
// directory is an empty replacement — through the real stores and a real bus. The unassignment says the
|
||||||
|
// data stays; the self-check's next run retires it (kept, listed by cleanup), and when the module comes
|
||||||
|
// up on another machine with an empty directory while the full one waits retired, that is urgent.
|
||||||
|
func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
||||||
|
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
||||||
|
house, err := catalogue.ParseManifest([]byte(houseManifest))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
register(t, open, house)
|
||||||
|
if _, err := assign(ctx, open, "laptop", "house"); err == nil {
|
||||||
|
t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up")
|
||||||
|
}
|
||||||
|
for _, node := range []string{"laptop", "anchor"} {
|
||||||
|
if _, err := assign(ctx, open, node, "keeper"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := assign(ctx, open, "laptop", "house"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
conn := onATestBus(t)
|
||||||
|
js, err := broker.Dial(os.Getenv("MESH_TEST_NATS"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
laptop, anchor := &fakeHolder{}, &fakeHolder{}
|
||||||
|
laptop.serve(t, conn, "laptop")
|
||||||
|
anchor.serve(t, conn, "anchor")
|
||||||
|
now := time.Now()
|
||||||
|
heard := &watchdogs{last: &signalFacts{now: now, machines: []machineFacts{
|
||||||
|
{name: "laptop", lastHeard: now}, {name: "anchor", lastHeard: now}}}}
|
||||||
|
d := &doctor{open: open, js: js, watchdogs: heard}
|
||||||
|
var d13 probe
|
||||||
|
for _, p := range probeRegistry {
|
||||||
|
if p.ID == probeDataID {
|
||||||
|
d13 = p
|
||||||
|
}
|
||||||
|
}
|
||||||
|
run := func() []conditions.Observation {
|
||||||
|
t.Helper()
|
||||||
|
probing := context.WithValue(ctx, probeAsksKey{}, d13)
|
||||||
|
obs, err := probeData(probing, d)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return obs
|
||||||
|
}
|
||||||
|
|
||||||
|
laptop.set(measuredHouse("/var/lib/house/config", 2<<30, now))
|
||||||
|
if obs := run(); len(obs) != 0 {
|
||||||
|
t.Fatalf("a measured, backed-up item raised %+v", obs)
|
||||||
|
}
|
||||||
|
r, err := inv.DataOf(ctx, "laptop", "house", "config")
|
||||||
|
if err != nil || r.Path != "/var/lib/house/config" || r.Size == nil || *r.Size != 2<<30 || r.LastBackup == nil {
|
||||||
|
t.Fatalf("what the holder measured was not kept: %+v, %v", r, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
said, err := unassign(ctx, open, "laptop", "house")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, "house's config (/var/lib/house/config, 2.0 GB) stays where it is") {
|
||||||
|
t.Fatalf("the unassignment does not say the data stays:\n%s", said)
|
||||||
|
}
|
||||||
|
laptop.set()
|
||||||
|
run()
|
||||||
|
r, err = inv.DataOf(ctx, "laptop", "house", "config")
|
||||||
|
if err != nil || !r.Retired() || r.Path != "/var/lib/house/config" {
|
||||||
|
t.Fatalf("unassigned, the irreplaceable item is not kept retired: %+v, %v", r, err)
|
||||||
|
}
|
||||||
|
records, _ := inv.Data(ctx)
|
||||||
|
if rows := retiredData(records, time.Now()); len(rows) != 1 || rows[0].Path != "/var/lib/house/config" {
|
||||||
|
t.Fatalf("cleanup list: %+v", rows)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Assigned on the anchor, onto an empty directory.
|
||||||
|
if _, err := assign(ctx, open, "anchor", "house"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
anchor.set(measuredHouse("/var/lib/house/config", 300<<10, time.Now()))
|
||||||
|
obs := findingsByKind(run())
|
||||||
|
o, ok := obs[kindEmptyReplacement]
|
||||||
|
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "laptop" {
|
||||||
|
t.Fatalf("an empty replacement of a module's data was not urgent: %+v", obs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Data on redundant storage: the array it is on is watched, one condition per array as loud as the most
|
||||||
|
// precious item on it; an item said to be on redundancy and found on plain storage is said; an item
|
||||||
|
// whose path is gone is said.
|
||||||
|
func TestTheArrayUnderDataIsWatched(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}],
|
||||||
|
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array, no room to copy"},
|
||||||
|
{"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array, no room to copy"}]}}`
|
||||||
|
shelf := shelfFor(t, media)
|
||||||
|
sick := false
|
||||||
|
on := func(item string, healthy *bool) inventory.DataRecord {
|
||||||
|
return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable", Owned: false,
|
||||||
|
Path: "/tank/" + item, Size: bytesOf(40 << 40), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now),
|
||||||
|
Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: healthy, Said: "pool 'tank' is DEGRADED"}}
|
||||||
|
}
|
||||||
|
got := dataFindings([]inventory.DataRecord{on("films", &sick), on("shows", &sick)}, nil, shelf, nil, nil, now)
|
||||||
|
if len(got) != 1 || got[0].Kind != kindArrayDegraded || got[0].Severity != conditions.Urgent ||
|
||||||
|
!strings.Contains(got[0].Summary, "media/films, media/shows") {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
well := true
|
||||||
|
if got := dataFindings([]inventory.DataRecord{on("films", &well)}, nil, shelf, nil, nil, now); len(got) != 0 {
|
||||||
|
t.Fatalf("a healthy array raised %+v", got)
|
||||||
|
}
|
||||||
|
plain := on("films", nil)
|
||||||
|
plain.Redundancy = nil
|
||||||
|
if o := findingsByKind(dataFindings([]inventory.DataRecord{plain}, nil, shelf, nil, nil, now))[kindProtectionMissing]; o.Severity != conditions.Urgent {
|
||||||
|
t.Fatalf("redundancy said and not found: %+v", o)
|
||||||
|
}
|
||||||
|
gone := on("films", &well)
|
||||||
|
gone.MeasureError, gone.Size = "/tank/films does not exist", bytesOf(0)
|
||||||
|
if o := findingsByKind(dataFindings([]inventory.DataRecord{gone}, map[string]int64{gone.Key(): 40 << 40}, shelf, nil, nil, now))[kindDataMissing]; o.Severity != conditions.Urgent {
|
||||||
|
t.Fatalf("a vanished library: %+v", o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What a consumer keeps with its provider as irreplaceable holds the provider's item to that class:
|
||||||
|
// the photo site's objects make the object store's data an urgent matter.
|
||||||
|
func TestKeptByHoldsTheProvidersItemToTheConsumersClass(t *testing.T) {
|
||||||
|
objects := `{"module":"objects","version":"1","provides":[{"name":"s3-bucket","scope":"mesh"}],"grants":{"s3-bucket":"${dir:g}"},
|
||||||
|
"data":{"own":[{"id":"data","path":"${dir:data}","class":"valuable"}],"consumers":{"s3-bucket":{"class":"valuable","in":"data"}}},
|
||||||
|
"resources":[{"id":"g","type":"directory","mode":"0700"},{"id":"data","type":"directory","mode":"0700"}]}`
|
||||||
|
photos := `{"module":"photos","version":"1","requires":["s3-bucket"],"data":{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}}`
|
||||||
|
shelf := shelfFor(t, objects, photos)
|
||||||
|
bindings := []inventory.Binding{{Machine: "anchor", Consumer: "photos", Provision: "s3-bucket",
|
||||||
|
Provider: catalogue.Chosen{Node: "anchor", Module: "objects"}}}
|
||||||
|
upgraded, keptBy := keptByClasses(bindings, shelf)
|
||||||
|
if upgraded["anchor/objects/data"] != "irreplaceable" || keptBy["objects/mesh_anchor_photos"] != "irreplaceable" {
|
||||||
|
t.Fatalf("upgraded %v, kept by %v", upgraded, keptBy)
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
r := inventory.DataRecord{Machine: "anchor", Module: "objects", Item: "data", Class: "valuable", Owned: true,
|
||||||
|
Size: bytesOf(10 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), MeasuredAt: when(now), LastBackup: when(now.Add(-72 * time.Hour))}
|
||||||
|
if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, upgraded, now))[kindBackupStale]; o.Severity != conditions.Urgent {
|
||||||
|
t.Fatalf("the photos' store without a backup: %+v", o)
|
||||||
|
}
|
||||||
|
if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now))[kindBackupStale]; o.Severity != conditions.Warning {
|
||||||
|
t.Fatalf("a valuable store without a backup: %+v", o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Items measured from one dataset's counters share its size: a shrink of the dataset is one condition
|
||||||
|
// naming every item on it, not one per item; and a partial walk's lower bound is never compared.
|
||||||
|
func TestADatasetShrinksOnceAndAPartialSizeIsNeverCompared(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}],
|
||||||
|
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array","measure":"dataset"},
|
||||||
|
{"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array","measure":"dataset"}]}}`
|
||||||
|
shelf := shelfFor(t, media, houseManifest)
|
||||||
|
well := true
|
||||||
|
on := func(item string, size int64) inventory.DataRecord {
|
||||||
|
return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable",
|
||||||
|
Size: bytesOf(size), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now),
|
||||||
|
Precision: "dataset tank/media: its whole size",
|
||||||
|
Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: &well}}
|
||||||
|
}
|
||||||
|
films, shows := on("films", 30<<40), on("shows", 30<<40)
|
||||||
|
peaks := map[string]int64{films.Key(): 90 << 40, shows.Key(): 90 << 40}
|
||||||
|
got := dataFindings([]inventory.DataRecord{films, shows}, peaks, shelf, nil, nil, now)
|
||||||
|
if len(got) != 1 || got[0].Kind != kindDataShrank || got[0].Severity != conditions.Urgent ||
|
||||||
|
!strings.Contains(got[0].Summary, "media/films, media/shows") {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
partial := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
|
||||||
|
Size: bytesOf(1 << 20), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now), LastWrite: when(now),
|
||||||
|
LastBackup: when(now), Precision: "partial: measured partially"}
|
||||||
|
if got := dataFindings([]inventory.DataRecord{partial}, map[string]int64{partial.Key(): 1 << 30}, shelf, nil, nil, now); len(got) != 0 {
|
||||||
|
t.Fatalf("a partial size was compared: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -102,6 +102,14 @@ var probeRegistry = []probe{
|
|||||||
{ID: probeBindingsID, Asserts: "every consumer of a provision that keeps its data is bound where it was last " +
|
{ID: probeBindingsID, Asserts: "every consumer of a provision that keeps its data is bound where it was last " +
|
||||||
"sent, or moves by a pin", From: "issue 273, ADR 0232", Kind: kindBindingMoved,
|
"sent, or moves by a pin", From: "issue 273, ADR 0232", Kind: kindBindingMoved,
|
||||||
Raises: []string{kindBindingKept, kindBindingMoving}, Phase: 2, run: probeBindings},
|
Raises: []string{kindBindingKept, kindBindingMoving}, Phase: 2, run: probeBindings},
|
||||||
|
{ID: probeDataID, Asserts: "every item of data a machine declares is measured, is there, holds what it held, is " +
|
||||||
|
"written where it should be, is backed up within its bound or sits on healthy redundant storage, and is no " +
|
||||||
|
"empty replacement of a copy kept elsewhere; what a machine no longer declares that is irreplaceable or " +
|
||||||
|
"valuable is retired, not forgotten", From: "issue 273, ADR 0233",
|
||||||
|
Kind: kindDataShrank, Raises: []string{kindEmptyReplacement, kindDataHeldTwice, kindDataQuiet, kindBackupStale,
|
||||||
|
kindDataUnmeasured, kindDataMissing, kindArrayDegraded, kindProtectionMissing, kindCleanupWaiting},
|
||||||
|
Phase: 2, run: probeData,
|
||||||
|
Asks: []broker.SeatVerb{{Seat: "node-backup", Verb: "backed-up"}}},
|
||||||
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -170,6 +170,9 @@ func run() error {
|
|||||||
return retireCommand(ctx, args[1:])
|
return retireCommand(ctx, args[1:])
|
||||||
case "cleanup":
|
case "cleanup":
|
||||||
return cleanupCommand(ctx, args[1:])
|
return cleanupCommand(ctx, args[1:])
|
||||||
|
// The data every machine declares, as the self-check last found it (novox/hq ADR 0233).
|
||||||
|
case "data":
|
||||||
|
return dataCommand(ctx, args[1:])
|
||||||
case "version":
|
case "version":
|
||||||
fmt.Println(version)
|
fmt.Println(version)
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -258,10 +258,21 @@ func cleanupCommand(ctx context.Context, args []string) error {
|
|||||||
}
|
}
|
||||||
defer open.Close()
|
defer open.Close()
|
||||||
return onTheBus(func(conn *nats.Conn) error {
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
return deleteOlderThan(ctx, open.inventory, conn, *olderThan, *confirm, f, time.Now())
|
return deleteOlderThan(ctx, open, conn, *olderThan, *confirm, f, time.Now())
|
||||||
})
|
})
|
||||||
case *olderThan == 0 && len(rest) == 3 && !*confirm:
|
case *olderThan == 0 && len(rest) == 3 && !*confirm:
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
return onTheBus(func(conn *nats.Conn) error {
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
|
// A module's own retired data, when the mesh holds such an item (novox/hq ADR 0233); a
|
||||||
|
// provider's retired consumer otherwise.
|
||||||
|
if r, err := open.inventory.DataOf(ctx, rest[0], rest[1], rest[2]); err == nil && r.DeletedAt == nil &&
|
||||||
|
r.RetiredAt != nil {
|
||||||
|
return deleteRetiredData(ctx, conn, open, r, f)
|
||||||
|
}
|
||||||
return deleteRetired(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, rest[2], f)
|
return deleteRetired(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, rest[2], f)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -285,6 +296,10 @@ type retiredRow struct {
|
|||||||
Why string `json:"why,omitempty"`
|
Why string `json:"why,omitempty"`
|
||||||
// Access is "kept" for a consumer of a mark-only provider: retired on record, still reachable.
|
// Access is "kept" for a consumer of a mark-only provider: retired on record, still reachable.
|
||||||
Access string `json:"access,omitempty"`
|
Access string `json:"access,omitempty"`
|
||||||
|
// Path and Class are a module's own retired data's (Kind own-data, novox/hq ADR 0233): where it is
|
||||||
|
// kept on its machine, and its class. Consumer is then the item.
|
||||||
|
Path string `json:"path,omitempty"`
|
||||||
|
Class string `json:"class,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// retiredListing is every provider's retired consumers, and the providers that could not say.
|
// retiredListing is every provider's retired consumers, and the providers that could not say.
|
||||||
@@ -299,7 +314,15 @@ func gatherRetired(ctx context.Context, inv *inventory.Inventory, conn *nats.Con
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return retiredListing{}, err
|
return retiredListing{}, err
|
||||||
}
|
}
|
||||||
return retiredOf(ctx, conn, instances, now), nil
|
listing := retiredOf(ctx, conn, instances, now)
|
||||||
|
// And every module's own data retired on its machine (novox/hq ADR 0233).
|
||||||
|
records, err := inv.Data(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return retiredListing{}, err
|
||||||
|
}
|
||||||
|
listing.Retired = append(listing.Retired, retiredData(records, now)...)
|
||||||
|
sort.SliceStable(listing.Retired, func(i, j int) bool { return listing.Retired[i].AgeDays > listing.Retired[j].AgeDays })
|
||||||
|
return listing, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func retiredOf(ctx context.Context, conn *nats.Conn, instances []providerInstance, now time.Time) retiredListing {
|
func retiredOf(ctx context.Context, conn *nats.Conn, instances []providerInstance, now time.Time) retiredListing {
|
||||||
@@ -332,7 +355,7 @@ func printRetired(l retiredListing, asJSON bool) error {
|
|||||||
return printJSON(l)
|
return printJSON(l)
|
||||||
}
|
}
|
||||||
if len(l.Retired) == 0 {
|
if len(l.Retired) == 0 {
|
||||||
fmt.Println("no provider holds a retired consumer")
|
fmt.Println("no provider holds a retired consumer, and no machine holds retired data")
|
||||||
}
|
}
|
||||||
for _, r := range l.Retired {
|
for _, r := range l.Retired {
|
||||||
age := "age unknown"
|
age := "age unknown"
|
||||||
@@ -346,6 +369,11 @@ func printRetired(l retiredListing, asJSON bool) error {
|
|||||||
if r.Access == "kept" {
|
if r.Access == "kept" {
|
||||||
kind += " [MARK ONLY: access kept until deleted]"
|
kind += " [MARK ONLY: access kept until deleted]"
|
||||||
}
|
}
|
||||||
|
if r.Kind == retiredDataKind {
|
||||||
|
fmt.Printf("%s on %s: its own %s, %s, at %s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer,
|
||||||
|
r.Class, r.Path, age, sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why))
|
||||||
|
continue
|
||||||
|
}
|
||||||
fmt.Printf("%s on %s: %s%s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer, kind, age,
|
fmt.Printf("%s on %s: %s%s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer, kind, age,
|
||||||
sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why))
|
sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why))
|
||||||
}
|
}
|
||||||
@@ -391,16 +419,16 @@ func deleteRetired(ctx context.Context, conn *nats.Conn, p providerInstance, con
|
|||||||
|
|
||||||
// deleteOlderThan lists every consumer retired more than days ago, and deletes them only with confirm.
|
// deleteOlderThan lists every consumer retired more than days ago, and deletes them only with confirm.
|
||||||
// One whose age the provider cannot say is never in it.
|
// One whose age the provider cannot say is never in it.
|
||||||
func deleteOlderThan(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, days int, confirm bool,
|
func deleteOlderThan(ctx context.Context, open *stores, conn *nats.Conn, days int, confirm bool,
|
||||||
f handActFlags, now time.Time) error {
|
f handActFlags, now time.Time) error {
|
||||||
listing, err := gatherRetired(ctx, inv, conn, now)
|
listing, err := gatherRetired(ctx, open.inventory, conn, now)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return deleteFrom(ctx, conn, listing, days, confirm, f)
|
return deleteFrom(ctx, conn, open, listing, days, confirm, f)
|
||||||
}
|
}
|
||||||
|
|
||||||
func deleteFrom(ctx context.Context, conn *nats.Conn, listing retiredListing, days int, confirm bool, f handActFlags) error {
|
func deleteFrom(ctx context.Context, conn *nats.Conn, open *stores, listing retiredListing, days int, confirm bool, f handActFlags) error {
|
||||||
var due []retiredRow
|
var due []retiredRow
|
||||||
unknown := 0
|
unknown := 0
|
||||||
for _, r := range listing.Retired {
|
for _, r := range listing.Retired {
|
||||||
@@ -431,7 +459,16 @@ func deleteFrom(ctx context.Context, conn *nats.Conn, listing retiredListing, da
|
|||||||
}
|
}
|
||||||
var failed []string
|
var failed []string
|
||||||
for _, r := range due {
|
for _, r := range due {
|
||||||
if err := deleteRetired(ctx, conn, providerInstance{Node: r.Node, Module: r.Module}, r.Consumer, f); err != nil {
|
var err error
|
||||||
|
if r.Kind == retiredDataKind {
|
||||||
|
var rec inventory.DataRecord
|
||||||
|
if rec, err = open.inventory.DataOf(ctx, r.Node, r.Module, r.Consumer); err == nil {
|
||||||
|
err = deleteRetiredData(ctx, conn, open, rec, f)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
err = deleteRetired(ctx, conn, providerInstance{Node: r.Node, Module: r.Module}, r.Consumer, f)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
failed = append(failed, err.Error())
|
failed = append(failed, err.Error())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -380,12 +380,12 @@ func TestNatsCleanupDeletesOnlyTheNamedRetiredConsumer(t *testing.T) {
|
|||||||
t.Fatalf("%+v", listing)
|
t.Fatalf("%+v", listing)
|
||||||
}
|
}
|
||||||
fake.deleted = nil
|
fake.deleted = nil
|
||||||
said = printed(t, func() error { return deleteFrom(t.Context(), conn, listing, 30, false, whyFlags(t, "tidy")) })
|
said = printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, false, whyFlags(t, "tidy")) })
|
||||||
if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") ||
|
if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") ||
|
||||||
strings.Contains(said, "young") {
|
strings.Contains(said, "young") {
|
||||||
t.Fatalf("deleted %v; said %s", fake.deleted, said)
|
t.Fatalf("deleted %v; said %s", fake.deleted, said)
|
||||||
}
|
}
|
||||||
printed(t, func() error { return deleteFrom(t.Context(), conn, listing, 30, true, whyFlags(t, "tidy")) })
|
printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, true, whyFlags(t, "tidy")) })
|
||||||
if !slices.Equal(fake.deleted, []string{"old"}) {
|
if !slices.Equal(fake.deleted, []string{"old"}) {
|
||||||
t.Fatalf("confirmed, deleted %v", fake.deleted)
|
t.Fatalf("confirmed, deleted %v", fake.deleted)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -499,6 +499,15 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
return argv, nil
|
return argv, nil
|
||||||
}
|
}
|
||||||
return []string{"cleanup", "list", "--json"}, nil
|
return []string{"cleanup", "list", "--json"}, nil
|
||||||
|
case "data":
|
||||||
|
argv := []string{"data", "--json"}
|
||||||
|
if m := str("machine"); m != "" {
|
||||||
|
argv = append(argv, "--machine", m)
|
||||||
|
}
|
||||||
|
if on("retired") {
|
||||||
|
argv = append(argv, "--retired")
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
case "doctor":
|
case "doctor":
|
||||||
which := 0
|
which := 0
|
||||||
argv := []string{"doctor"}
|
argv := []string{"doctor"}
|
||||||
@@ -598,7 +607,7 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
|
|
||||||
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
|
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
|
||||||
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true,
|
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true,
|
||||||
"hand-acts": true, "durations": true, "conditions": true, "doctor": true, "retire": true, "cleanup": true}
|
"hand-acts": true, "durations": true, "conditions": true, "doctor": true, "retire": true, "cleanup": true, "data": true}
|
||||||
|
|
||||||
// repairingCommand names a command line that repairs by hand, and so says why: a push, a plan stopped
|
// repairingCommand names a command line that repairs by hand, and so says why: a push, a plan stopped
|
||||||
// or closed, a consumer re-made (novox/hq to-be 45 §7). Empty for any other.
|
// or closed, a consumer re-made (novox/hq to-be 45 §7). Empty for any other.
|
||||||
|
|||||||
@@ -276,6 +276,7 @@ var accountedFlags = map[string]map[string]string{
|
|||||||
"conditions": {"json": "set by the verb: the answer is data"},
|
"conditions": {"json": "set by the verb: the answer is data"},
|
||||||
"retire": {"json": "set by the verb: the answer is data"},
|
"retire": {"json": "set by the verb: the answer is data"},
|
||||||
"cleanup": {"json": "set by the verb: the answer is data"},
|
"cleanup": {"json": "set by the verb: the answer is data"},
|
||||||
|
"data": {"json": "set by the verb: the answer is data"},
|
||||||
"conditions history": {"json": "set by the verb: the answer is data"},
|
"conditions history": {"json": "set by the verb: the answer is data"},
|
||||||
"conditions show": {"json": "set by the verb: the answer is data"},
|
"conditions show": {"json": "set by the verb: the answer is data"},
|
||||||
"healers": {"json": "set by the verb: the answer is data"},
|
"healers": {"json": "set by the verb: the answer is data"},
|
||||||
|
|||||||
@@ -133,7 +133,11 @@ type SeatVerb struct{ Seat, Verb string }
|
|||||||
// to the probe registry is the reason they cannot drift** — a probe that calls a verb its grant does
|
// to the probe registry is the reason they cannot drift** — a probe that calls a verb its grant does
|
||||||
// not name is refused by the bus on every run (found live on 2026-10-06: D8 timed out on each
|
// not name is refused by the bus on every run (found live on 2026-10-06: D8 timed out on each
|
||||||
// machine, refused). Asked of any machine (`.*`), read-only verbs, nothing else of the seat.
|
// machine, refused). Asked of any machine (`.*`), read-only verbs, nothing else of the seat.
|
||||||
var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}}
|
//
|
||||||
|
// D13 reads every machine's backup holder: what it measured of the data declared there (novox/hq ADR
|
||||||
|
// 0233).
|
||||||
|
var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"},
|
||||||
|
{Seat: "node-backup", Verb: "backed-up"}}
|
||||||
|
|
||||||
// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
|
// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
|
||||||
// holder's machine (novox/hq ADR 0219): `paused.<node>`, the build agent saying whether it takes work.
|
// holder's machine (novox/hq ADR 0219): `paused.<node>`, the build agent saying whether it takes work.
|
||||||
|
|||||||
+1
-1
@@ -24,7 +24,7 @@ accounts {
|
|||||||
jetstream: enabled
|
jetstream: enabled
|
||||||
users = [
|
users = [
|
||||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
|
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
|
||||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||||
allow_responses: { max: 1, ttl: "1m" }
|
allow_responses: { max: 1, ttl: "1m" }
|
||||||
} }
|
} }
|
||||||
|
|||||||
@@ -5,35 +5,10 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
// A store provider says how its data is backed up (novox/hq ADR 0214); a provider of something that
|
// A backup contribution written by hand still names its module's own directories; one it does not
|
||||||
// holds nothing does not have to.
|
// declare is refused where it is written rather than reaching the holder as the literal text. (The
|
||||||
func TestAStoreProviderWithoutABackupIsRefusedByTheCheck(t *testing.T) {
|
// catalogue check refuses a hand-written backup line at all since ADR 0233; parsing still reads one,
|
||||||
bare, err := ParseManifest([]byte(`{"module":"pg","version":"1",
|
// because a module on the shelf was written before.)
|
||||||
"provides":[{"name":"postgres-database","scope":"mesh"}]}`))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parsing refused it, and the rule is the check's: %v", err)
|
|
||||||
}
|
|
||||||
if problems := CheckBackup(bare); len(problems) != 1 || !strings.Contains(problems[0], "node-backup") {
|
|
||||||
t.Fatalf("a store with no backup passed the check: %v", problems)
|
|
||||||
}
|
|
||||||
backed, err := ParseManifest([]byte(`{"module":"pg","version":"1",
|
|
||||||
"provides":[{"name":"postgres-database","scope":"mesh"}],
|
|
||||||
"resources":[{"id":"dumps","type":"directory","mode":"0700"}],
|
|
||||||
"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:dumps}"}]}`))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if problems := CheckBackup(backed); len(problems) != 0 {
|
|
||||||
t.Fatalf("a store that contributes a backup was refused: %v", problems)
|
|
||||||
}
|
|
||||||
route, _ := ParseManifest([]byte(`{"module":"r","version":"1","provides":[{"name":"route","scope":"mesh"}]}`))
|
|
||||||
if problems := CheckBackup(route); len(problems) != 0 {
|
|
||||||
t.Fatalf("a route was asked for a backup: %v", problems)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A backup contribution names its module's own directories; one it does not declare is refused
|
|
||||||
// where it is written rather than reaching the holder as the literal text.
|
|
||||||
func TestABackupNamingAnUndeclaredDirectoryIsRefused(t *testing.T) {
|
func TestABackupNamingAnUndeclaredDirectoryIsRefused(t *testing.T) {
|
||||||
_, err := ParseManifest([]byte(`{"module":"pg","version":"1",
|
_, err := ParseManifest([]byte(`{"module":"pg","version":"1",
|
||||||
"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:nowhere}"}]}`))
|
"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:nowhere}"}]}`))
|
||||||
@@ -42,9 +17,9 @@ func TestABackupNamingAnUndeclaredDirectoryIsRefused(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The holder receives every module's backup lines with each module's own directories filled, each
|
// A module on the shelf from before ADR 0233, which writes its backup lines by hand and declares no
|
||||||
// module's under a comment naming it — and a kind written in a shell's grammar is left untouched.
|
// data, is still backed up: its lines are placed as written, each module's under a comment naming it.
|
||||||
func TestBackupContributionsArePlacedWithTheirModulesDirectories(t *testing.T) {
|
func TestHandWrittenBackupLinesOfAnOlderModuleArePlaced(t *testing.T) {
|
||||||
pg, err := ParseManifest([]byte(`{"module":"pg","version":"1",
|
pg, err := ParseManifest([]byte(`{"module":"pg","version":"1",
|
||||||
"resources":[{"id":"dumps","type":"directory","path":"/srv/pg/dumps","mode":"0700"}],
|
"resources":[{"id":"dumps","type":"directory","path":"/srv/pg/dumps","mode":"0700"}],
|
||||||
"contributions":[{"seat":"node-backup","kind":"backup","content":"run pg-dump-all\npath ${dir:dumps}"}]}`))
|
"contributions":[{"seat":"node-backup","kind":"backup","content":"run pg-dump-all\npath ${dir:dumps}"}]}`))
|
||||||
@@ -57,7 +32,7 @@ func TestBackupContributionsArePlacedWithTheirModulesDirectories(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
placed, err := seatContributions([]Manifest{pg, mail}, BackupSeat, "backup", Rendering{})
|
placed, err := seatContributions([]Manifest{pg, mail}, BackupSeat, "backup", Rendering{}, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -87,15 +87,18 @@ func TestNoCatalogueManifestNamesAnInstallation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestEveryCatalogueStoreSaysHowItIsBackedUp is ADR 0214's check over the real catalogue: a module
|
// TestEveryCatalogueModuleDeclaresItsData is ADR 0233's check over the real catalogue (it replaced ADR
|
||||||
// providing a store contributes a backup to node-backup, so a store added is a store backed up.
|
// 0214's store list): every provider that grants says what it keeps for its consumers, nothing writes a
|
||||||
func TestEveryCatalogueStoreSaysHowItIsBackedUp(t *testing.T) {
|
// backup line by hand, every directory a container writes is declared, and every irreplaceable item is
|
||||||
|
// backed up — so a store added is a store backed up, without a list of stores to keep in step.
|
||||||
|
func TestEveryCatalogueModuleDeclaresItsData(t *testing.T) {
|
||||||
root := catalogueRoot(t)
|
root := catalogueRoot(t)
|
||||||
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||||
if err != nil || len(found) == 0 {
|
if err != nil || len(found) == 0 {
|
||||||
t.Fatalf("no manifests under %s: %v", root, err)
|
t.Fatalf("no manifests under %s: %v", root, err)
|
||||||
}
|
}
|
||||||
stores := 0
|
shelf := Shelf{}
|
||||||
|
granting := 0
|
||||||
for _, p := range found {
|
for _, p := range found {
|
||||||
raw, err := os.ReadFile(p)
|
raw, err := os.ReadFile(p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -105,18 +108,16 @@ func TestEveryCatalogueStoreSaysHowItIsBackedUp(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
continue // TestEveryCatalogueManifestParses says why
|
continue // TestEveryCatalogueManifestParses says why
|
||||||
}
|
}
|
||||||
for _, o := range m.Provides {
|
if len(m.Grants) > 0 {
|
||||||
if storeProvisions[o.Name] {
|
granting++
|
||||||
stores++
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, problem := range CheckBackup(m) {
|
|
||||||
t.Error(problem)
|
|
||||||
}
|
}
|
||||||
|
shelf[m.Module] = m
|
||||||
}
|
}
|
||||||
if stores == 0 {
|
for _, problem := range DataProblems(shelf) {
|
||||||
t.Fatal("no module in the catalogue provides a store, so this proved nothing")
|
t.Error(problem)
|
||||||
|
}
|
||||||
|
if granting == 0 {
|
||||||
|
t.Fatal("no module in the catalogue grants a provision, so this proved nothing")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,791 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module declares the data it holds, and the mesh protects and watches it from that declaration
|
||||||
|
// (novox/hq ADR 0233).
|
||||||
|
//
|
||||||
|
// **One section, `data`, for every kind of data a module keeps:** its own, by directory — a store's
|
||||||
|
// files, a mail spool, an application's uploads — or by an operator's path it was given; what it keeps
|
||||||
|
// for its consumers, by the provision they reach it through; and what of its own lives with a
|
||||||
|
// provider, by the provision it requires. Each entry has a class — how precious it is — and, for its
|
||||||
|
// own data, how it is protected; everything the mesh does is derived from those, never written per
|
||||||
|
// module:
|
||||||
|
//
|
||||||
|
// - a binding to a consumer's data does not move (ADR 0232) where the provision's class keeps data;
|
||||||
|
// - the backup holder's lines are composed from it — a module no longer writes them by hand;
|
||||||
|
// - what an unassignment leaves behind of an irreplaceable or valuable item is retired, listed by
|
||||||
|
// `cleanup list` and deleted only by `cleanup delete` (ADR 0230);
|
||||||
|
// - the self-check measures every item and says when one shrinks, disappears, stops being written,
|
||||||
|
// goes without its backup, sits on a degraded array, or is replaced by an empty copy of itself —
|
||||||
|
// urgent for what is irreplaceable, a warning for what is valuable.
|
||||||
|
|
||||||
|
// The classes: how precious the data is. A fixed vocabulary, ranked by the operator (2026-10-06): a
|
||||||
|
// class is what the protections are derived from, so a new one is a decision, not a manifest's choice.
|
||||||
|
const (
|
||||||
|
// ClassIrreplaceable is what must never be lost: the operator names it (the media library, the
|
||||||
|
// photo sites' storage). Protected by a backup or by a declared redundancy — one is required —
|
||||||
|
// retired rather than removed, and every alert about it is urgent.
|
||||||
|
ClassIrreplaceable = "irreplaceable"
|
||||||
|
// ClassValuable is anybody's work that would be painful to lose: in the nightly backup by default,
|
||||||
|
// retired rather than removed, and every alert about it a warning.
|
||||||
|
ClassValuable = "valuable"
|
||||||
|
// ClassRebuildable can be made again from something kept elsewhere — a clone, an index, a
|
||||||
|
// download, a night's dump — at a cost in time, not in data. In the nightly backup by default;
|
||||||
|
// forgotten when its module goes, and never alerted on.
|
||||||
|
ClassRebuildable = "rebuildable"
|
||||||
|
// ClassCache may be emptied at any moment with nothing lost but speed: never backed up, never
|
||||||
|
// measured, never alerted on.
|
||||||
|
ClassCache = "cache"
|
||||||
|
// ClassNone is a provision that keeps nothing of its consumers' (consumers only): a resolver, a
|
||||||
|
// certificate authority, an artifact store.
|
||||||
|
ClassNone = "none"
|
||||||
|
)
|
||||||
|
|
||||||
|
// classRank orders the classes by how precious they are, so the stricter of two is chosen.
|
||||||
|
var classRank = map[string]int{ClassNone: 0, ClassCache: 1, ClassRebuildable: 2, ClassValuable: 3, ClassIrreplaceable: 4}
|
||||||
|
|
||||||
|
// StricterClass is the more precious of two classes.
|
||||||
|
func StricterClass(a, b string) string {
|
||||||
|
if classRank[b] > classRank[a] {
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retires is whether a class's data is retired, not forgotten, when its machine no longer declares it.
|
||||||
|
func Retires(class string) bool { return class == ClassIrreplaceable || class == ClassValuable }
|
||||||
|
|
||||||
|
// Watched is whether a class's data raises conditions: irreplaceable and valuable.
|
||||||
|
func Watched(class string) bool { return Retires(class) }
|
||||||
|
|
||||||
|
// DefaultBackupWithin is how old the last good backup of an item may be before the self-check says so
|
||||||
|
// (novox/hq ADR 0214: a machine with data and no good backup in 48 hours).
|
||||||
|
const DefaultBackupWithin = 48 * time.Hour
|
||||||
|
|
||||||
|
// Data is a manifest's `data` section.
|
||||||
|
type Data struct {
|
||||||
|
// Own is the data this module keeps itself.
|
||||||
|
Own []DataItem `json:"own,omitempty"`
|
||||||
|
// Consumers is what it keeps for its consumers, per provision it grants.
|
||||||
|
Consumers map[string]ConsumerData `json:"consumers,omitempty"`
|
||||||
|
// KeptBy is what of its own lives with the provider of a provision it requires — its rows, its
|
||||||
|
// objects — and how precious that is. The provider's protections follow the stricter of its own
|
||||||
|
// class for its consumers and this.
|
||||||
|
KeptBy map[string]KeptData `json:"kept-by,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// DataItem is one piece of a module's own data.
|
||||||
|
type DataItem struct {
|
||||||
|
// ID names it within the module: what `data`, `cleanup` and a condition call it.
|
||||||
|
ID string `json:"id"`
|
||||||
|
// Path is one of the module's directories, `${dir:<id>}`, or an operator's path it was given,
|
||||||
|
// `${access:<id>}`, or a path beneath either. Never a machine path (novox/hq ADR 0112).
|
||||||
|
Path string `json:"path"`
|
||||||
|
Class string `json:"class"`
|
||||||
|
// Backup is how it is copied: "copy" (the holder reads it as it stands), "none", or a dump — a
|
||||||
|
// command that writes a consistent copy into another item, which is copied. Unsaid, anything but a
|
||||||
|
// cache is copied, unless it says it is protected by redundancy instead.
|
||||||
|
Backup *DataBackup `json:"backup,omitempty"`
|
||||||
|
// Redundancy says it is protected by the redundancy of the storage it lives on rather than by a
|
||||||
|
// copy, and why that is enough: the media library on an array there is no room to copy. The
|
||||||
|
// backup holder then watches that array, and a degraded one is said.
|
||||||
|
Redundancy string `json:"redundancy,omitempty"`
|
||||||
|
// Within is how old its last good backup may be; unsaid, DefaultBackupWithin.
|
||||||
|
Within string `json:"within,omitempty"`
|
||||||
|
// Measure is how the backup holder measures it: `walk` (the default — every file, at most daily and
|
||||||
|
// bounded, for small items), `dataset` (a ZFS dataset's own counters, hourly, nothing walked) or
|
||||||
|
// `shallow` (its top-level entries only, no size). A large item never says walk.
|
||||||
|
Measure string `json:"measure,omitempty"`
|
||||||
|
// Active is how long it may go unwritten before that is a fault — for data something is
|
||||||
|
// expected to write all the time. Unsaid, a quiet item is not a fault.
|
||||||
|
Active string `json:"active,omitempty"`
|
||||||
|
// Why is a line for the reviewer: why this class.
|
||||||
|
Why string `json:"why,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConsumerData is what a provider keeps for the consumers of one provision.
|
||||||
|
type ConsumerData struct {
|
||||||
|
Class string `json:"class"`
|
||||||
|
// In is where it lives: one of the module's own items (whose protection covers it), or a
|
||||||
|
// provision this module requires (whose provider keeps it, as its consumer). Unsaid only for none
|
||||||
|
// and cache.
|
||||||
|
In string `json:"in,omitempty"`
|
||||||
|
Why string `json:"why,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// KeptData is how precious what a module keeps with a provider is.
|
||||||
|
type KeptData struct {
|
||||||
|
Class string `json:"class"`
|
||||||
|
Why string `json:"why,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// DataBackup is how an item is copied.
|
||||||
|
type DataBackup struct {
|
||||||
|
Copy bool
|
||||||
|
None bool
|
||||||
|
// Dump is the command writing a consistent copy into the item Into.
|
||||||
|
Dump string
|
||||||
|
Into string
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalJSON reads "copy", "none" or {"dump": "...", "into": "<item>"}.
|
||||||
|
func (b *DataBackup) UnmarshalJSON(raw []byte) error {
|
||||||
|
var word string
|
||||||
|
if err := json.Unmarshal(raw, &word); err == nil {
|
||||||
|
switch word {
|
||||||
|
case "copy":
|
||||||
|
*b = DataBackup{Copy: true}
|
||||||
|
case "none":
|
||||||
|
*b = DataBackup{None: true}
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}, not %q", word)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var full struct {
|
||||||
|
Dump string `json:"dump"`
|
||||||
|
Into string `json:"into"`
|
||||||
|
}
|
||||||
|
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||||
|
dec.DisallowUnknownFields()
|
||||||
|
if err := dec.Decode(&full); err != nil {
|
||||||
|
return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}: %w", err)
|
||||||
|
}
|
||||||
|
*b = DataBackup{Dump: full.Dump, Into: full.Into}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalJSON writes it back in the form it was written.
|
||||||
|
func (b DataBackup) MarshalJSON() ([]byte, error) {
|
||||||
|
switch {
|
||||||
|
case b.Copy:
|
||||||
|
return json.Marshal("copy")
|
||||||
|
case b.None:
|
||||||
|
return json.Marshal("none")
|
||||||
|
}
|
||||||
|
return json.Marshal(struct {
|
||||||
|
Dump string `json:"dump"`
|
||||||
|
Into string `json:"into"`
|
||||||
|
}{b.Dump, b.Into})
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsDump is whether the item is copied by a dump.
|
||||||
|
func (b *DataBackup) IsDump() bool { return b != nil && b.Dump != "" }
|
||||||
|
|
||||||
|
// BackedUp is whether the holder keeps restore points of this item: anything but a cache by default —
|
||||||
|
// the nightly backup is the standard plan — unless it says "none", or says it is protected by
|
||||||
|
// redundancy and says nothing of a backup.
|
||||||
|
func (it DataItem) BackedUp() bool {
|
||||||
|
switch {
|
||||||
|
case it.Backup == nil:
|
||||||
|
return it.Class != ClassCache && it.Redundancy == ""
|
||||||
|
case it.Backup.None:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Protection is how the item is protected, in one word: "backup", "redundancy", both joined by "+",
|
||||||
|
// or "none".
|
||||||
|
func (it DataItem) Protection() string {
|
||||||
|
var by []string
|
||||||
|
if it.BackedUp() {
|
||||||
|
by = append(by, "backup")
|
||||||
|
}
|
||||||
|
if it.Redundancy != "" {
|
||||||
|
by = append(by, "redundancy")
|
||||||
|
}
|
||||||
|
if len(by) == 0 {
|
||||||
|
return "none"
|
||||||
|
}
|
||||||
|
return strings.Join(by, "+")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The ways an item is measured.
|
||||||
|
const (
|
||||||
|
MeasureWalk = "walk"
|
||||||
|
MeasureDataset = "dataset"
|
||||||
|
MeasureShallow = "shallow"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MeasuredBy is how the item is measured, with the default applied.
|
||||||
|
func (it DataItem) MeasuredBy() string {
|
||||||
|
if it.Measure == "" {
|
||||||
|
return MeasureWalk
|
||||||
|
}
|
||||||
|
return it.Measure
|
||||||
|
}
|
||||||
|
|
||||||
|
// OwnedByModule is whether the item is in one of the module's own directories — the mesh's to retire —
|
||||||
|
// rather than an operator's path it was given, which the mesh never retires and never deletes.
|
||||||
|
func (it DataItem) OwnedByModule() bool { return strings.HasPrefix(it.Path, "${dir:") }
|
||||||
|
|
||||||
|
// BackupWithin is the item's bound on its last good backup.
|
||||||
|
func (it DataItem) BackupWithin() time.Duration {
|
||||||
|
if d, err := ParseDataDuration(it.Within); err == nil && d > 0 {
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
return DefaultBackupWithin
|
||||||
|
}
|
||||||
|
|
||||||
|
// ActiveWithin is how long the item may go unwritten; zero when quiet is not a fault.
|
||||||
|
func (it DataItem) ActiveWithin() time.Duration {
|
||||||
|
d, _ := ParseDataDuration(it.Active)
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseDataDuration reads "48h", "90m" or "7d"; empty is zero.
|
||||||
|
func ParseDataDuration(s string) (time.Duration, error) {
|
||||||
|
s = strings.TrimSpace(s)
|
||||||
|
if s == "" {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
if days, ok := strings.CutSuffix(s, "d"); ok {
|
||||||
|
n, err := strconv.Atoi(days)
|
||||||
|
if err != nil || n <= 0 {
|
||||||
|
return 0, fmt.Errorf("%q is not a number of days", s)
|
||||||
|
}
|
||||||
|
return time.Duration(n) * 24 * time.Hour, nil
|
||||||
|
}
|
||||||
|
d, err := time.ParseDuration(s)
|
||||||
|
if err != nil || d <= 0 {
|
||||||
|
return 0, fmt.Errorf("%q is not a duration (48h, 90m, 7d)", s)
|
||||||
|
}
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DataItems is the module's own data, in the order declared.
|
||||||
|
func (m Manifest) DataItems() []DataItem {
|
||||||
|
if m.Data == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return m.Data.Own
|
||||||
|
}
|
||||||
|
|
||||||
|
// DataItem is one own item by id.
|
||||||
|
func (m Manifest) DataItem(id string) (DataItem, bool) {
|
||||||
|
for _, it := range m.DataItems() {
|
||||||
|
if it.ID == id {
|
||||||
|
return it, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return DataItem{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConsumerDataOf is what this module says it keeps for a provision's consumers, and whether it says.
|
||||||
|
func (m Manifest) ConsumerDataOf(provision string) (ConsumerData, bool) {
|
||||||
|
if m.Data == nil {
|
||||||
|
return ConsumerData{}, false
|
||||||
|
}
|
||||||
|
c, ok := m.Data.Consumers[provision]
|
||||||
|
return c, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// KeptByOf is how precious what this module keeps with a provision's provider is, and whether it says.
|
||||||
|
func (m Manifest) KeptByOf(provision string) (KeptData, bool) {
|
||||||
|
if m.Data == nil {
|
||||||
|
return KeptData{}, false
|
||||||
|
}
|
||||||
|
k, ok := m.Data.KeptBy[provision]
|
||||||
|
return k, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepsByClass is whether a class keeps something a binding must not move away from.
|
||||||
|
func keepsByClass(class string) bool {
|
||||||
|
return class == ClassIrreplaceable || class == ClassValuable || class == ClassRebuildable
|
||||||
|
}
|
||||||
|
|
||||||
|
// dataID is what an item's id may be: it is a token in a condition's key and a word on a line.
|
||||||
|
var dataID = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
|
||||||
|
|
||||||
|
// dataPathRef is an item's path: one of the module's directories or accesses, and optionally a path
|
||||||
|
// beneath it.
|
||||||
|
var dataPathRef = regexp.MustCompile(`^\$\{(dir|access):([a-z0-9][a-z0-9-]*)\}(/[^\s$]*)?$`)
|
||||||
|
|
||||||
|
// dataProblems is what is wrong with the `data` section itself, from the manifest alone: judged at
|
||||||
|
// registration as every other per-manifest problem is. Whether a module declares what it should is
|
||||||
|
// the catalogue check's (DataProblems), because a module already running was written before it.
|
||||||
|
func (m Manifest) dataProblems() []string {
|
||||||
|
if m.Data == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var problems []string
|
||||||
|
say := func(format string, args ...any) {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s's data: ", m.Module)+fmt.Sprintf(format, args...))
|
||||||
|
}
|
||||||
|
dirs := map[string]bool{}
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if fmt.Sprint(r["type"]) == "directory" {
|
||||||
|
dirs[fmt.Sprint(r["id"])] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
accesses := map[string]bool{}
|
||||||
|
for _, a := range m.Accesses {
|
||||||
|
if a.ID != "" {
|
||||||
|
accesses[a.ID] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ids := map[string]DataItem{}
|
||||||
|
paths := map[string]string{}
|
||||||
|
for i, it := range m.Data.Own {
|
||||||
|
label := it.ID
|
||||||
|
if label == "" {
|
||||||
|
label = fmt.Sprintf("item %d", i+1)
|
||||||
|
}
|
||||||
|
if !dataID.MatchString(it.ID) {
|
||||||
|
say("%s has no usable id: lower-case letters, digits and dashes", label)
|
||||||
|
} else if _, twice := ids[it.ID]; twice {
|
||||||
|
say("%s is declared twice", it.ID)
|
||||||
|
}
|
||||||
|
ids[it.ID] = it
|
||||||
|
ref := dataPathRef.FindStringSubmatch(it.Path)
|
||||||
|
switch {
|
||||||
|
case ref == nil:
|
||||||
|
say("%s's path %q is not one of the module's directories or accesses: ${dir:<id>} or ${access:<id>}, "+
|
||||||
|
"or a path beneath one (a definition names no machine path, novox/hq ADR 0112)", label, it.Path)
|
||||||
|
case ref[1] == "dir" && !dirs[ref[2]]:
|
||||||
|
say("%s's path names ${dir:%s}, and %s declares no directory %q", label, ref[2], m.Module, ref[2])
|
||||||
|
case ref[1] == "access" && !accesses[ref[2]]:
|
||||||
|
say("%s's path names ${access:%s}, and %s declares no access %q", label, ref[2], m.Module, ref[2])
|
||||||
|
case strings.Contains(it.Path, ".."):
|
||||||
|
say("%s's path %q climbs out of its directory", label, it.Path)
|
||||||
|
}
|
||||||
|
if other, twice := paths[it.Path]; twice && it.Path != "" {
|
||||||
|
say("%s and %s name the same path %s", other, label, it.Path)
|
||||||
|
}
|
||||||
|
paths[it.Path] = label
|
||||||
|
switch it.Class {
|
||||||
|
case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache:
|
||||||
|
case ClassNone:
|
||||||
|
say("%s is class none, which only a provision keeping nothing of its consumers' is; own data "+
|
||||||
|
"that is disposable is cache", label)
|
||||||
|
default:
|
||||||
|
say("%s's class %q is not one the mesh protects by: irreplaceable, valuable, rebuildable or cache",
|
||||||
|
label, it.Class)
|
||||||
|
}
|
||||||
|
if it.Class == ClassIrreplaceable && !it.BackedUp() && strings.TrimSpace(it.Redundancy) == "" {
|
||||||
|
say("%s is irreplaceable and is neither backed up nor said to be protected by redundancy; the only "+
|
||||||
|
"copy of something is protected one way or the other (novox/hq ADR 0233) — copy it, dump it into "+
|
||||||
|
"another item, or say `redundancy` with why that is enough", label)
|
||||||
|
}
|
||||||
|
if it.Class == ClassCache && it.Backup != nil && !it.Backup.None {
|
||||||
|
say("%s is a cache and asks to be backed up; a cache is disposable, or it is not a cache", label)
|
||||||
|
}
|
||||||
|
if it.Class == ClassCache && it.Redundancy != "" {
|
||||||
|
say("%s is a cache and says it is protected by redundancy; a cache is not protected", label)
|
||||||
|
}
|
||||||
|
switch it.Measure {
|
||||||
|
case "", MeasureWalk, MeasureDataset, MeasureShallow:
|
||||||
|
default:
|
||||||
|
say("%s's measure %q is walk, dataset or shallow", label, it.Measure)
|
||||||
|
}
|
||||||
|
if _, err := ParseDataDuration(it.Within); err != nil {
|
||||||
|
say("%s's within: %v", label, err)
|
||||||
|
}
|
||||||
|
if _, err := ParseDataDuration(it.Active); err != nil {
|
||||||
|
say("%s's active: %v", label, err)
|
||||||
|
}
|
||||||
|
if it.Within != "" && !it.BackedUp() {
|
||||||
|
say("%s states within, and is not backed up", label)
|
||||||
|
}
|
||||||
|
if it.Active != "" && !Watched(it.Class) {
|
||||||
|
say("%s is %s and expects writes; only irreplaceable and valuable data is watched", label, it.Class)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Dumps go into an item of the same module, declared, and not into themselves.
|
||||||
|
for _, it := range m.Data.Own {
|
||||||
|
if it.Backup == nil || it.Backup.Copy || it.Backup.None {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch into, ok := ids[it.Backup.Into]; {
|
||||||
|
case strings.TrimSpace(it.Backup.Dump) == "":
|
||||||
|
say("%s's backup names no dump command", it.ID)
|
||||||
|
case it.Backup.Into == "":
|
||||||
|
say("%s's dump says no item it writes into", it.ID)
|
||||||
|
case !ok:
|
||||||
|
say("%s's dump writes into %q, which is not one of the module's data items", it.ID, it.Backup.Into)
|
||||||
|
case into.ID == it.ID:
|
||||||
|
say("%s's dump writes into itself; a dump is copied from where it lands", it.ID)
|
||||||
|
case into.Class == ClassCache:
|
||||||
|
say("%s's dump writes into %s, a cache; what is copied must not be disposable", it.ID, into.ID)
|
||||||
|
}
|
||||||
|
if it.Backup.Dump != "" {
|
||||||
|
declared := map[string]string{}
|
||||||
|
for d := range dirs {
|
||||||
|
declared[d] = ""
|
||||||
|
}
|
||||||
|
if _, err := dirFill(it.Backup.Dump, declared, m.Module); err != nil {
|
||||||
|
say("%s's dump: %v", it.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
provided := map[string]bool{}
|
||||||
|
for _, o := range m.Provides {
|
||||||
|
provided[o.Name] = true
|
||||||
|
}
|
||||||
|
wants := map[string]bool{}
|
||||||
|
for _, w := range m.Wants() {
|
||||||
|
wants[w] = true
|
||||||
|
}
|
||||||
|
for _, provision := range sortedKeys(m.Data.Consumers) {
|
||||||
|
c := m.Data.Consumers[provision]
|
||||||
|
if !provided[provision] {
|
||||||
|
say("says what it keeps for the consumers of %q, which it does not provide", provision)
|
||||||
|
}
|
||||||
|
switch c.Class {
|
||||||
|
case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache, ClassNone:
|
||||||
|
default:
|
||||||
|
say("its consumers' %s is class %q; one of irreplaceable, valuable, rebuildable, cache or none", provision, c.Class)
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case c.Class == ClassNone && c.In != "":
|
||||||
|
say("its consumers' %s keeps nothing and says where it is kept (%s)", provision, c.In)
|
||||||
|
case keepsByClass(c.Class) && c.In == "":
|
||||||
|
say("its consumers' %s is %s and says nowhere it lives: `in` names one of the module's items, or a "+
|
||||||
|
"provision it requires", provision, c.Class)
|
||||||
|
case c.In != "":
|
||||||
|
if own, ok := ids[c.In]; ok {
|
||||||
|
if c.Class == ClassIrreplaceable && !own.BackedUp() && own.Redundancy == "" {
|
||||||
|
say("its consumers' %s is irreplaceable and lives in %s, which is not protected", provision, c.In)
|
||||||
|
}
|
||||||
|
if classRank[own.Class] < classRank[c.Class] {
|
||||||
|
say("its consumers' %s is %s and lives in %s, which is only %s", provision, c.Class, c.In, own.Class)
|
||||||
|
}
|
||||||
|
} else if !wants[c.In] {
|
||||||
|
say("its consumers' %s lives in %q, which is neither one of its data items nor a provision it "+
|
||||||
|
"requires", provision, c.In)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, provision := range sortedKeys(m.Data.KeptBy) {
|
||||||
|
k := m.Data.KeptBy[provision]
|
||||||
|
if !wants[provision] {
|
||||||
|
say("says it keeps data with the provider of %q, which it does not require", provision)
|
||||||
|
}
|
||||||
|
switch k.Class {
|
||||||
|
case ClassIrreplaceable, ClassValuable, ClassRebuildable, ClassCache:
|
||||||
|
default:
|
||||||
|
say("what it keeps with %s is class %q; one of irreplaceable, valuable, rebuildable or cache", provision, k.Class)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return problems
|
||||||
|
}
|
||||||
|
|
||||||
|
// KeepsConsumerData is whether this module, providing a provision, keeps what each consumer writes
|
||||||
|
// there (novox/hq ADR 0232, ADR 0233): whether a consumer bound to it is bound to its data.
|
||||||
|
//
|
||||||
|
// **What the data section says, it gets**: irreplaceable, valuable or rebuildable keeps; cache and none
|
||||||
|
// do not — a cache lost in a move costs speed, not data. Before the section, an offer said it with
|
||||||
|
// `keeps-consumer-data`, which is still read; unsaid in both, a provider that grants each consumer a
|
||||||
|
// credential of its own keeps that consumer's data (ADR 0232 §1). The catalogue check refuses the
|
||||||
|
// unsaid case for a provider that grants (DataProblems), so the inference is what an older definition
|
||||||
|
// on the shelf gets, never a new one.
|
||||||
|
func (m Manifest) KeepsConsumerData(provision string) bool {
|
||||||
|
if c, ok := m.ConsumerDataOf(provision); ok {
|
||||||
|
return keepsByClass(c.Class)
|
||||||
|
}
|
||||||
|
for _, o := range m.Provides {
|
||||||
|
if o.Name == provision && o.KeepsConsumerData != nil {
|
||||||
|
return *o.KeepsConsumerData
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, grants := m.Grants[provision]
|
||||||
|
return grants
|
||||||
|
}
|
||||||
|
|
||||||
|
// NeedsBackupHolder is whether a module's data needs the machine's backup holder to be there: it keeps
|
||||||
|
// something irreplaceable — backed up by the holder, or protected by an array the holder watches. A
|
||||||
|
// module keeping only valuable or rebuildable data is backed up where a holder is, and refused nowhere
|
||||||
|
// for want of one: the standard plan, not a requirement.
|
||||||
|
func (m Manifest) NeedsBackupHolder() bool {
|
||||||
|
for _, it := range m.DataItems() {
|
||||||
|
if it.Class == ClassIrreplaceable {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- derived: the backup holder's lines ---------------------------------------------------------
|
||||||
|
|
||||||
|
// The node-backup seat's kinds (novox/hq to-be 43, ADR 0233): `backup` is what to run and which paths
|
||||||
|
// to keep, `data` every item with its class and protection, for the holder to measure and watch.
|
||||||
|
const (
|
||||||
|
BackupKindBackup = "backup"
|
||||||
|
BackupKindData = "data"
|
||||||
|
)
|
||||||
|
|
||||||
|
// derivedContributions is what a module's data section gives the backup holder: its backup lines and
|
||||||
|
// its items. Every item is listed, so a valuable one on a machine is measured whether or not it is
|
||||||
|
// copied; a cache is listed and not measured.
|
||||||
|
func (m Manifest) derivedContributions() []SeatContribution {
|
||||||
|
items := m.DataItems()
|
||||||
|
if len(items) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var lines []string
|
||||||
|
kept := map[string]bool{}
|
||||||
|
keep := func(path string) {
|
||||||
|
if !kept[path] {
|
||||||
|
kept[path] = true
|
||||||
|
lines = append(lines, "path "+path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, it := range items {
|
||||||
|
if !it.BackedUp() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if it.Backup.IsDump() {
|
||||||
|
lines = append(lines, "run "+strings.TrimSpace(it.Backup.Dump))
|
||||||
|
if into, ok := m.DataItem(it.Backup.Into); ok {
|
||||||
|
keep(into.Path)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
keep(it.Path)
|
||||||
|
}
|
||||||
|
var described []string
|
||||||
|
for _, it := range items {
|
||||||
|
covered := "-"
|
||||||
|
switch {
|
||||||
|
case it.Backup.IsDump():
|
||||||
|
if into, ok := m.DataItem(it.Backup.Into); ok {
|
||||||
|
covered = into.Path
|
||||||
|
}
|
||||||
|
case it.BackedUp():
|
||||||
|
covered = it.Path
|
||||||
|
}
|
||||||
|
described = append(described, fmt.Sprintf("item %s %s %s %s %s %s", it.ID, it.Class, it.Path, covered,
|
||||||
|
it.Protection(), it.MeasuredBy()))
|
||||||
|
}
|
||||||
|
var out []SeatContribution
|
||||||
|
if len(lines) > 0 {
|
||||||
|
out = append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindBackup, Content: strings.Join(lines, "\n") + "\n"})
|
||||||
|
}
|
||||||
|
return append(out, SeatContribution{Seat: BackupSeat, Kind: BackupKindData, Content: strings.Join(described, "\n") + "\n"})
|
||||||
|
}
|
||||||
|
|
||||||
|
// allContributions is every contribution a module makes to a seat's holder: what it wrote, and what
|
||||||
|
// its data section derives. **One list**: a module that declares its data has its backup lines
|
||||||
|
// composed from it, and a backup line it still writes by hand is not placed — the catalogue check
|
||||||
|
// refuses it — so the holder never copies two lists that disagree.
|
||||||
|
func (m Manifest) allContributions() []SeatContribution {
|
||||||
|
if m.Data == nil {
|
||||||
|
return m.Contributions
|
||||||
|
}
|
||||||
|
derived := m.derivedContributions()
|
||||||
|
out := make([]SeatContribution, 0, len(m.Contributions)+len(derived))
|
||||||
|
for _, c := range m.Contributions {
|
||||||
|
if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
return append(out, derived...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- the catalogue check ------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// DataProblems is what the catalogue check refuses about the data a module declares (novox/hq ADR
|
||||||
|
// 0233), judged over the manifests given together:
|
||||||
|
//
|
||||||
|
// - a provider that grants a provision says what it keeps for that provision's consumers;
|
||||||
|
// - nobody says it on the offer any more (`keeps-consumer-data`): the data section is the one place;
|
||||||
|
// - nobody writes a backup line by hand: it is derived from the data section;
|
||||||
|
// - a directory a container writes, mounted whole, is a data item of some class;
|
||||||
|
// - consumer data said to live in a required provision lives where that provision's provider keeps
|
||||||
|
// its consumers' data, as preciously, when the provider is given;
|
||||||
|
// - data a consumer keeps with a provider as irreplaceable is protected there, when the provider is
|
||||||
|
// given.
|
||||||
|
//
|
||||||
|
// **The check's, not registration's**, as ADR 0214's backup rule was: a module already on the shelf
|
||||||
|
// was written before the rule, and refusing it there would refuse the very providers whose data the
|
||||||
|
// rule protects. Each module meets it where it is written.
|
||||||
|
func DataProblems(shelf Shelf) []string {
|
||||||
|
var problems []string
|
||||||
|
for _, name := range shelfOrder(shelf) {
|
||||||
|
m := shelf[name]
|
||||||
|
for _, provision := range sortedKeys(m.Grants) {
|
||||||
|
if _, said := m.ConsumerDataOf(provision); !said {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s grants %s and does not say what it keeps for its consumers: data.consumers.%s with a "+
|
||||||
|
"class — irreplaceable, valuable, rebuildable, cache, or none (novox/hq ADR 0233)", name, provision, provision))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, o := range m.Provides {
|
||||||
|
if o.KeepsConsumerData != nil {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s says keeps-consumer-data on its offer of %s; it is said once, in data.consumers.%s, with a "+
|
||||||
|
"class (novox/hq ADR 0233)", name, o.Name, o.Name))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i, c := range m.Contributions {
|
||||||
|
if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s's contribution %d is a backup line written by hand; backups are derived from the data "+
|
||||||
|
"section — declare the data, with its class and how it is protected (novox/hq ADR 0233)", name, i+1))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, dir := range writtenDirectories(m) {
|
||||||
|
if !coversDirectory(m, dir) {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s mounts its directory %q into a container to be written, and declares no data in it: "+
|
||||||
|
"data.own with a class — cache if it is disposable (novox/hq ADR 0233)", name, dir))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if m.Data == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, provision := range sortedKeys(m.Data.Consumers) {
|
||||||
|
c := m.Data.Consumers[provision]
|
||||||
|
if c.In == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, own := m.DataItem(c.In); own {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, pname := range shelfOrder(shelf) {
|
||||||
|
p := shelf[pname]
|
||||||
|
pc, said := p.ConsumerDataOf(c.In)
|
||||||
|
if !said || !providesName(p, c.In) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if classRank[pc.Class] < classRank[c.Class] {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s keeps its consumers' %s, %s, in %s — and %s keeps its consumers' %s as %s, "+
|
||||||
|
"so it is not protected as %s", name, provision, c.Class, c.In, pname, c.In, pc.Class, c.Class))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, provision := range sortedKeys(m.Data.KeptBy) {
|
||||||
|
k := m.Data.KeptBy[provision]
|
||||||
|
if k.Class != ClassIrreplaceable {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, pname := range shelfOrder(shelf) {
|
||||||
|
p := shelf[pname]
|
||||||
|
pc, said := p.ConsumerDataOf(provision)
|
||||||
|
if !said || !providesName(p, provision) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !keepsByClass(pc.Class) {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s keeps irreplaceable data with %s, and %s keeps its consumers' %s as %s — nothing of it is "+
|
||||||
|
"protected there", name, provision, pname, provision, pc.Class))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if in, own := p.DataItem(pc.In); own && !in.BackedUp() && in.Redundancy == "" {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s keeps irreplaceable data with %s, and %s keeps it in %s, which is neither backed up nor "+
|
||||||
|
"on declared redundancy", name, provision, pname, pc.In))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return problems
|
||||||
|
}
|
||||||
|
|
||||||
|
func providesName(m Manifest, provision string) bool {
|
||||||
|
for _, o := range m.Provides {
|
||||||
|
if o.Name == provision {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// writtenDirectories are the module's directories a container mounts whole and may write: a volume
|
||||||
|
// `${dir:<id>}:<target>` without `:ro`, or a directory stated by an absolute path mounted the same
|
||||||
|
// way. A file beneath a directory, or a read-only mount, is configuration the mesh wrote.
|
||||||
|
func writtenDirectories(m Manifest) []string {
|
||||||
|
absolute := map[string]string{}
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if fmt.Sprint(r["type"]) != "directory" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if p, ok := r["path"].(string); ok && strings.HasPrefix(p, "/") {
|
||||||
|
absolute[strings.TrimRight(p, "/")] = fmt.Sprint(r["id"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if fmt.Sprint(r["type"]) != "container" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
vols, _ := r["volumes"].([]any)
|
||||||
|
for _, v := range vols {
|
||||||
|
s, _ := v.(string)
|
||||||
|
mount := volumeMount.FindStringSubmatch(s)
|
||||||
|
if mount == nil || volumeReadOnly(mount[3]) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
src := strings.TrimRight(mount[1], "/")
|
||||||
|
if ref := dirPlain.FindStringSubmatch(src); ref != nil {
|
||||||
|
seen[ref[1]] = true
|
||||||
|
} else if id, ok := absolute[src]; ok {
|
||||||
|
seen[id] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out := make([]string, 0, len(seen))
|
||||||
|
for id := range seen {
|
||||||
|
out = append(out, id)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// volumeMount is a container's volume, `<source>:<target>[:<options>]`, its source possibly a
|
||||||
|
// `${dir:<id>}` — whose own colon is not the separator.
|
||||||
|
var volumeMount = regexp.MustCompile(`^(\$\{(?:dir|access):[a-z0-9][a-z0-9-]*\}[^:]*|[^:]+):([^:]+)(?::(.*))?$`)
|
||||||
|
|
||||||
|
// volumeReadOnly is whether a volume's options mount it read-only.
|
||||||
|
func volumeReadOnly(options string) bool {
|
||||||
|
for _, o := range strings.Split(options, ",") {
|
||||||
|
if strings.TrimSpace(o) == "ro" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// dirPlain is a whole directory, `${dir:<id>}` and nothing after it.
|
||||||
|
var dirPlain = regexp.MustCompile(`^\$\{dir:([a-z0-9][a-z0-9-]*)\}$`)
|
||||||
|
|
||||||
|
// coversDirectory is whether a data item names the directory, a path within it, or a directory it
|
||||||
|
// is placed beneath.
|
||||||
|
func coversDirectory(m Manifest, dir string) bool {
|
||||||
|
parents := map[string]string{}
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if fmt.Sprint(r["type"]) != "directory" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if p, ok := r["path"].(string); ok {
|
||||||
|
if ref := dirRef.FindStringSubmatch(p); ref != nil && strings.HasPrefix(p, "${dir:") {
|
||||||
|
parents[fmt.Sprint(r["id"])] = ref[1]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, it := range m.DataItems() {
|
||||||
|
ref := dataPathRef.FindStringSubmatch(it.Path)
|
||||||
|
if ref == nil || ref[1] != "dir" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for d, hops := dir, 0; d != "" && hops < 4; d, hops = parents[d], hops+1 {
|
||||||
|
if ref[2] == d {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,227 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A store as a module now declares it (novox/hq ADR 0233): its own data by directory with a class and a
|
||||||
|
// dump, and what it keeps for its consumers.
|
||||||
|
const declaredStore = `{"module":"pg","version":"1",
|
||||||
|
"provides":[{"name":"postgres-database","scope":"mesh"}],
|
||||||
|
"grants":{"postgres-database":"${dir:grants}"},
|
||||||
|
"data":{
|
||||||
|
"own":[
|
||||||
|
{"id":"store","path":"${dir:store}","class":"irreplaceable","backup":{"dump":"dump-all > ${dir:dumps}/all","into":"dumps"},"active":"1d"},
|
||||||
|
{"id":"dumps","path":"${dir:dumps}","class":"rebuildable","backup":"none"}],
|
||||||
|
"consumers":{"postgres-database":{"class":"irreplaceable","in":"store"}}},
|
||||||
|
"resources":[
|
||||||
|
{"id":"grants","type":"directory","mode":"0700"},
|
||||||
|
{"id":"store","type":"directory","path":"/srv/store","mode":"0700"},
|
||||||
|
{"id":"dumps","type":"directory","path":"${dir:store}/dumps","mode":"0700"},
|
||||||
|
{"id":"server","type":"container","image":"x@sha256:aa","volumes":["${dir:store}:/var/lib/postgresql/data"]}]}`
|
||||||
|
|
||||||
|
func mustParse(t *testing.T, raw string) Manifest {
|
||||||
|
t.Helper()
|
||||||
|
m, err := ParseManifest([]byte(raw))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("refused: %v", err)
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestADeclaredStoreParsesAndPassesTheCheck(t *testing.T) {
|
||||||
|
m := mustParse(t, declaredStore)
|
||||||
|
if problems := DataProblems(Shelf{"pg": m}); len(problems) > 0 {
|
||||||
|
t.Fatalf("a store declaring its data was refused: %v", problems)
|
||||||
|
}
|
||||||
|
if !m.KeepsConsumerData("postgres-database") {
|
||||||
|
t.Fatal("an irreplaceable consumer class does not keep the consumer's data")
|
||||||
|
}
|
||||||
|
if it, _ := m.DataItem("store"); !it.BackedUp() || it.BackupWithin() != DefaultBackupWithin || it.ActiveWithin().Hours() != 24 {
|
||||||
|
t.Fatalf("the store item reads %+v", it)
|
||||||
|
}
|
||||||
|
if it, _ := m.DataItem("dumps"); it.BackedUp() {
|
||||||
|
t.Fatal("a rebuildable item that says none is backed up")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every rule of the section itself, refused at parse in the words of the module.
|
||||||
|
func TestTheDataSectionIsRefusedWhereItIsWrong(t *testing.T) {
|
||||||
|
for _, c := range []struct{ name, data, want string }{
|
||||||
|
{"a class it does not know", `{"own":[{"id":"a","path":"${dir:d}","class":"precious"}]}`, "not one the mesh protects by"},
|
||||||
|
{"an access it does not have", `{"own":[{"id":"a","path":"${access:media}","class":"irreplaceable","redundancy":"an array"}]}`, "declares no access"},
|
||||||
|
{"irreplaceable and protected by nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":"none"}]}`, "protected one way or the other"},
|
||||||
|
{"a cache on redundancy", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","redundancy":"an array"}]}`, "a cache is not protected"},
|
||||||
|
{"kept with something not required", `{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}`, "does not require"},
|
||||||
|
{"a measure it does not know", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","measure":"du"}]}`, "walk, dataset or shallow"},
|
||||||
|
{"a cache expecting writes", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","active":"1d"}]}`, "only irreplaceable and valuable"},
|
||||||
|
{"a machine path", `{"own":[{"id":"a","path":"/srv/a","class":"cache"}]}`, "names no machine path"},
|
||||||
|
{"a directory it does not declare", `{"own":[{"id":"a","path":"${dir:nowhere}","class":"cache"}]}`, "declares no directory"},
|
||||||
|
{"a cache backed up", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","backup":"copy"}]}`, "disposable"},
|
||||||
|
{"a dump into nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":{"dump":"x","into":"b"}}]}`, "not one of the module's data items"},
|
||||||
|
{"a dump into itself", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":{"dump":"x","into":"a"}}]}`, "into itself"},
|
||||||
|
{"a bad duration", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","within":"soon"}]}`, "not a duration"},
|
||||||
|
{"the same id twice", `{"own":[{"id":"a","path":"${dir:d}","class":"cache"},{"id":"a","path":"${dir:d}/x","class":"cache"}]}`, "declared twice"},
|
||||||
|
{"own data of class none", `{"own":[{"id":"a","path":"${dir:d}","class":"none"}]}`, "only a provision"},
|
||||||
|
{"consumers of something not provided", `{"consumers":{"s3-bucket":{"class":"cache"}}}`, "does not provide"},
|
||||||
|
{"consumer data that lives nowhere", `{"consumers":{"q":{"class":"irreplaceable"}}}`, "says nowhere it lives"},
|
||||||
|
{"consumer data in an item less precious", `{"own":[{"id":"a","path":"${dir:d}","class":"rebuildable"}],"consumers":{"q":{"class":"irreplaceable","in":"a"}}}`, "which is only rebuildable"},
|
||||||
|
{"consumer data in something unknown", `{"consumers":{"q":{"class":"rebuildable","in":"elsewhere"}}}`, "neither one of its data items"},
|
||||||
|
} {
|
||||||
|
raw := `{"module":"m","version":"1","provides":["q"],"resources":[{"id":"d","type":"directory","mode":"0700"}],"data":` + c.data + `}`
|
||||||
|
_, err := ParseManifest([]byte(raw))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), c.want) {
|
||||||
|
t.Errorf("%s: %v, want %q", c.name, err, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The catalogue check: a granting provider says what it keeps, says it in one place, writes no backup
|
||||||
|
// line by hand, and every directory a container writes is declared (novox/hq ADR 0233).
|
||||||
|
func TestTheCatalogueCheckAsksEveryModuleForItsData(t *testing.T) {
|
||||||
|
unsaid := mustParse(t, `{"module":"q","version":"1","provides":[{"name":"queue","scope":"mesh"}],
|
||||||
|
"grants":{"queue":"${dir:g}"},"resources":[{"id":"g","type":"directory","mode":"0700"}]}`)
|
||||||
|
onTheOffer := mustParse(t, `{"module":"r","version":"1","provides":[{"name":"resolver","scope":"mesh","keeps-consumer-data":false}]}`)
|
||||||
|
byHand := mustParse(t, `{"module":"h","version":"1","resources":[{"id":"d","type":"directory","mode":"0700"}],
|
||||||
|
"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:d}"}]}`)
|
||||||
|
writes := mustParse(t, `{"module":"w","version":"1","resources":[{"id":"d","type":"directory","mode":"0700"},
|
||||||
|
{"id":"c","type":"directory","mode":"0700"},
|
||||||
|
{"id":"app","type":"container","image":"x@sha256:aa","volumes":["${dir:d}:/data","${dir:c}:/etc/app:ro","${dir:d}/f:/f"]}]}`)
|
||||||
|
problems := strings.Join(DataProblems(Shelf{"q": unsaid, "r": onTheOffer, "h": byHand, "w": writes}), "\n")
|
||||||
|
for _, want := range []string{
|
||||||
|
"q grants queue and does not say what it keeps",
|
||||||
|
"r says keeps-consumer-data on its offer",
|
||||||
|
"h's contribution 1 is a backup line written by hand",
|
||||||
|
`w mounts its directory "d" into a container to be written`,
|
||||||
|
} {
|
||||||
|
if !strings.Contains(problems, want) {
|
||||||
|
t.Errorf("the check does not say %q:\n%s", want, problems)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(problems, `"c"`) {
|
||||||
|
t.Errorf("a read-only mount was taken for written data:\n%s", problems)
|
||||||
|
}
|
||||||
|
// The same module declaring the directory, as a cache, passes.
|
||||||
|
declared := mustParse(t, `{"module":"w","version":"1","data":{"own":[{"id":"d","path":"${dir:d}","class":"cache"}]},
|
||||||
|
"resources":[{"id":"d","type":"directory","mode":"0700"},
|
||||||
|
{"id":"app","type":"container","image":"x@sha256:aa","volumes":["${dir:d}:/data"]}]}`)
|
||||||
|
if p := DataProblems(Shelf{"w": declared}); len(p) > 0 {
|
||||||
|
t.Fatalf("a declared directory is still refused: %v", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Consumer data said to live in a provision the module requires is protected only as well as that
|
||||||
|
// provision's provider protects its consumers' data.
|
||||||
|
func TestConsumerDataInARequiredProvisionIsJudgedAgainstItsProvider(t *testing.T) {
|
||||||
|
idp := mustParse(t, `{"module":"idp","version":"1","requires":["postgres-database"],
|
||||||
|
"provides":[{"name":"oidc-client","scope":"mesh"}],"grants":{"oidc-client":"${dir:g}"},
|
||||||
|
"resources":[{"id":"g","type":"directory","mode":"0700"}],
|
||||||
|
"data":{"consumers":{"oidc-client":{"class":"irreplaceable","in":"postgres-database"}}}}`)
|
||||||
|
cachy := mustParse(t, strings.Replace(declaredStore, `"consumers":{"postgres-database":{"class":"irreplaceable","in":"store"}}`,
|
||||||
|
`"consumers":{"postgres-database":{"class":"cache"}}`, 1))
|
||||||
|
if p := strings.Join(DataProblems(Shelf{"idp": idp, "pg": cachy}), "\n"); !strings.Contains(p, "not protected as irreplaceable") {
|
||||||
|
t.Fatalf("irreplaceable data kept in a cache passed: %s", p)
|
||||||
|
}
|
||||||
|
if p := DataProblems(Shelf{"idp": idp, "pg": mustParse(t, declaredStore)}); len(p) > 0 {
|
||||||
|
t.Fatalf("refused against a provider that keeps its consumers' data: %v", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What a binding is sticky by follows the class (ADR 0232, generalised by 0233): a cache's consumers
|
||||||
|
// move freely, a store's do not; an older definition saying nothing still follows its grant.
|
||||||
|
func TestKeepingConsumerDataFollowsTheClass(t *testing.T) {
|
||||||
|
for class, keeps := range map[string]bool{"irreplaceable": true, "valuable": true, "rebuildable": true, "cache": false, "none": false} {
|
||||||
|
in := `,"in":"d"`
|
||||||
|
own := `"own":[{"id":"d","path":"${dir:d}","class":"irreplaceable"}],`
|
||||||
|
if !keeps {
|
||||||
|
in, own = "", ""
|
||||||
|
}
|
||||||
|
m := mustParse(t, `{"module":"p","version":"1","provides":[{"name":"q","scope":"mesh"}],"grants":{"q":"${dir:d}"},
|
||||||
|
"resources":[{"id":"d","type":"directory","mode":"0700"}],
|
||||||
|
"data":{`+own+`"consumers":{"q":{"class":"`+class+`"`+in+`}}}}`)
|
||||||
|
if m.KeepsConsumerData("q") != keeps {
|
||||||
|
t.Errorf("class %s keeps consumer data: %v, want %v", class, !keeps, keeps)
|
||||||
|
}
|
||||||
|
shelf := map[string]Manifest{"p": m}
|
||||||
|
if KeepsConsumerData(shelf, "q") != keeps {
|
||||||
|
t.Errorf("class %s: the provision by name keeps: %v", class, !keeps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
older := mustParse(t, `{"module":"p","version":"1","provides":[{"name":"q","scope":"mesh"}],"grants":{"q":"${dir:d}"},
|
||||||
|
"resources":[{"id":"d","type":"directory","mode":"0700"}]}`)
|
||||||
|
if !older.KeepsConsumerData("q") {
|
||||||
|
t.Fatal("an older definition that grants no longer keeps its consumers' data")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The backup holder's lines are derived: the dump runs and the directory it writes into is kept,
|
||||||
|
// anything but a cache is copied by default — the standard nightly plan — an item on redundancy is not
|
||||||
|
// copied, and every item is listed with its class and protection for the holder to measure and watch:
|
||||||
|
// directories filled, a home directory filled from the machine, an operator's path from where the
|
||||||
|
// assignment placed it. A backup line still written by hand beside a data section is not placed.
|
||||||
|
func TestTheBackupHoldersLinesAreDerivedFromTheData(t *testing.T) {
|
||||||
|
pg := mustParse(t, declaredStore)
|
||||||
|
pg.Contributions = append(pg.Contributions, SeatContribution{Seat: BackupSeat, Kind: "backup", Content: "path /stale"})
|
||||||
|
agent := mustParse(t, `{"module":"agent","version":"1",
|
||||||
|
"data":{"own":[{"id":"home","path":"${dir:home}","class":"valuable"},{"id":"tmp","path":"${dir:home}/tmp","class":"cache"}]},
|
||||||
|
"resources":[{"id":"home","type":"directory","path":"${machine:account-home}/.agent","mode":"0700"}]}`)
|
||||||
|
media := mustParse(t, `{"module":"media","version":"1",
|
||||||
|
"accesses":[{"id":"films","mode":"read"}],
|
||||||
|
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"on a redundant array; no room for a copy","measure":"dataset"},
|
||||||
|
{"id":"meta","path":"${dir:meta}","class":"rebuildable"}]},
|
||||||
|
"resources":[{"id":"meta","type":"directory","mode":"0700"}]}`)
|
||||||
|
facts := map[string]string{"account-home": "/home/op"}
|
||||||
|
with := Rendering{Settings: SettingsBy{"media": {{From: "node", Values: map[string]any{AccessesSetting: map[string]any{"films": "/tank/films"}}}}}}
|
||||||
|
backup, err := seatContributions([]Manifest{pg, agent, media}, BackupSeat, BackupKindBackup, with, facts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := "# agent\npath /home/op/.agent\n# media\npath /var/lib/media/meta\n# pg\nrun dump-all > /srv/store/dumps/all\npath /srv/store/dumps\n"
|
||||||
|
if backup != want {
|
||||||
|
t.Fatalf("backup lines:\n%s\nwant:\n%s", backup, want)
|
||||||
|
}
|
||||||
|
data, err := seatContributions([]Manifest{pg, agent, media}, BackupSeat, BackupKindData, with, facts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want = "# agent\nitem home valuable /home/op/.agent /home/op/.agent backup walk\nitem tmp cache /home/op/.agent/tmp - none walk\n" +
|
||||||
|
"# media\nitem films irreplaceable /tank/films - redundancy dataset\nitem meta rebuildable /var/lib/media/meta /var/lib/media/meta backup walk\n" +
|
||||||
|
"# pg\nitem store irreplaceable /srv/store /srv/store/dumps backup walk\nitem dumps rebuildable /srv/store/dumps - none walk\n"
|
||||||
|
if data != want {
|
||||||
|
t.Fatalf("data lines:\n%s\nwant:\n%s", data, want)
|
||||||
|
}
|
||||||
|
if _, err := seatContributions([]Manifest{agent}, BackupSeat, BackupKindData, Rendering{}, nil); err == nil {
|
||||||
|
t.Fatal("a home directory with no account on the machine reached the holder as a placeholder")
|
||||||
|
}
|
||||||
|
// What keeps something irreplaceable depends on the machine's backup holder — it backs it up or
|
||||||
|
// watches its array; valuable data alone is backed up where a holder is, and refused nowhere.
|
||||||
|
if !slices.Contains(DependsOn(media), BackupSeat) || !slices.Contains(DependsOn(pg), BackupSeat) ||
|
||||||
|
slices.Contains(DependsOn(agent), BackupSeat) {
|
||||||
|
t.Fatalf("depends: media %v, pg %v, agent %v", DependsOn(media), DependsOn(pg), DependsOn(agent))
|
||||||
|
}
|
||||||
|
if it, _ := media.DataItem("films"); it.OwnedByModule() || it.Protection() != "redundancy" {
|
||||||
|
t.Fatalf("an operator's path reads %+v", it)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Data a consumer keeps with a provider as irreplaceable must be protected there: the check refuses a
|
||||||
|
// provider that keeps its consumers' data as a cache, or in an item with no protection.
|
||||||
|
func TestIrreplaceableDataKeptWithAProviderMustBeProtectedThere(t *testing.T) {
|
||||||
|
photos := mustParse(t, `{"module":"photos","version":"1","requires":["s3-bucket"],
|
||||||
|
"data":{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}}`)
|
||||||
|
store := func(backup string) Manifest {
|
||||||
|
return mustParse(t, `{"module":"objects","version":"1","provides":[{"name":"s3-bucket","scope":"mesh"}],
|
||||||
|
"grants":{"s3-bucket":"${dir:g}"},
|
||||||
|
"data":{"own":[{"id":"data","path":"${dir:data}","class":"valuable","backup":"`+backup+`"}],
|
||||||
|
"consumers":{"s3-bucket":{"class":"valuable","in":"data"}}},
|
||||||
|
"resources":[{"id":"g","type":"directory","mode":"0700"},{"id":"data","type":"directory","mode":"0700"}]}`)
|
||||||
|
}
|
||||||
|
if p := DataProblems(Shelf{"photos": photos, "objects": store("copy")}); len(p) > 0 {
|
||||||
|
t.Fatalf("a provider backing its consumers' data up was refused: %v", p)
|
||||||
|
}
|
||||||
|
if p := strings.Join(DataProblems(Shelf{"photos": photos, "objects": store("none")}), "\n"); !strings.Contains(p, "neither backed up") {
|
||||||
|
t.Fatalf("irreplaceable photos in an unprotected store passed: %s", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -576,7 +576,7 @@ func contributionsInto(resource map[string]any, m Manifest, modules []Manifest,
|
|||||||
found := ofContributed.FindStringSubmatch(placeholder)
|
found := ofContributed.FindStringSubmatch(placeholder)
|
||||||
first, second, _ := strings.Cut(found[2], ":")
|
first, second, _ := strings.Cut(found[2], ":")
|
||||||
if found[1] == "contribution" {
|
if found[1] == "contribution" {
|
||||||
placed, err := seatContributions(modules, first, second, with)
|
placed, err := seatContributions(modules, first, second, with, facts)
|
||||||
if err != nil && failed == nil {
|
if err != nil && failed == nil {
|
||||||
failed = err
|
failed = err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -239,25 +239,6 @@ func (m Manifest) IdentityBoundOf(provision string) IdentityBound {
|
|||||||
return IdentityBound{}
|
return IdentityBound{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// KeepsConsumerData is whether this module, providing a provision, keeps what each consumer writes
|
|
||||||
// there (novox/hq ADR 0232): whether a consumer bound to it is bound to its data.
|
|
||||||
//
|
|
||||||
// **What an offer says, it gets.** Unsaid, it follows from whether the provider grants each consumer
|
|
||||||
// a credential of its own: a provider that does makes an account for every consumer — a role and its
|
|
||||||
// database, a key and its bucket, a client — and what the consumer writes under that account stays
|
|
||||||
// with that provider. One that grants nothing keeps nothing of anybody's: the resolver, a CA, the
|
|
||||||
// artifact store each answer any consumer alike, and moving a consumer between two of them loses
|
|
||||||
// nothing.
|
|
||||||
func (m Manifest) KeepsConsumerData(provision string) bool {
|
|
||||||
for _, o := range m.Provides {
|
|
||||||
if o.Name == provision && o.KeepsConsumerData != nil {
|
|
||||||
return *o.KeepsConsumerData
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_, grants := m.Grants[provision]
|
|
||||||
return grants
|
|
||||||
}
|
|
||||||
|
|
||||||
// KeepsConsumerData is whether a provision, by name, keeps its consumers' data across the
|
// KeepsConsumerData is whether a provision, by name, keeps its consumers' data across the
|
||||||
// catalogue: true when any module providing it at the mesh's scope does. **A property of the name**,
|
// catalogue: true when any module providing it at the mesh's scope does. **A property of the name**,
|
||||||
// as brokering is: two providers disagreeing would make the same binding sticky or free depending on
|
// as brokering is: two providers disagreeing would make the same binding sticky or free depending on
|
||||||
@@ -466,6 +447,12 @@ type Manifest struct {
|
|||||||
// (novox/hq ADR 0201). Not history — that is an event — and never a secret, sealed or not.
|
// (novox/hq ADR 0201). Not history — that is an event — and never a secret, sealed or not.
|
||||||
State []StateDeclaration `json:"state,omitempty"`
|
State []StateDeclaration `json:"state,omitempty"`
|
||||||
|
|
||||||
|
// Data is every kind of data this module keeps — its own, by directory, and what it keeps for
|
||||||
|
// its consumers, by provision — each with a class the mesh protects and watches it by (novox/hq
|
||||||
|
// ADR 0233). One list: the backup holder's lines, the bindings that do not move, what an
|
||||||
|
// unassignment retires and what the self-check measures are all derived from it.
|
||||||
|
Data *Data `json:"data,omitempty"`
|
||||||
|
|
||||||
// Reads are other modules' state this module reads and watches, each `<module>.<name>`
|
// Reads are other modules' state this module reads and watches, each `<module>.<name>`
|
||||||
// (novox/hq ADR 0201). Read-only: only the owner's instances write.
|
// (novox/hq ADR 0201). Read-only: only the owner's instances write.
|
||||||
Reads []string `json:"reads,omitempty"`
|
Reads []string `json:"reads,omitempty"`
|
||||||
@@ -2007,6 +1994,7 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
|||||||
problems = append(problems, m.shellProblems()...)
|
problems = append(problems, m.shellProblems()...)
|
||||||
problems = append(problems, m.contributionPlaceholderProblems()...)
|
problems = append(problems, m.contributionPlaceholderProblems()...)
|
||||||
problems = append(problems, m.seatContributionProblems()...)
|
problems = append(problems, m.seatContributionProblems()...)
|
||||||
|
problems = append(problems, m.dataProblems()...)
|
||||||
|
|
||||||
for i, r := range m.Resources {
|
for i, r := range m.Resources {
|
||||||
id, _ := r["id"].(string)
|
id, _ := r["id"].(string)
|
||||||
|
|||||||
@@ -23,46 +23,6 @@ const MessageBusSeat = "node-message-bus"
|
|||||||
// BackupSeat is the machine's backups (novox/hq ADR 0214, to-be 43).
|
// BackupSeat is the machine's backups (novox/hq ADR 0214, to-be 43).
|
||||||
const BackupSeat = "node-backup"
|
const BackupSeat = "node-backup"
|
||||||
|
|
||||||
// storeProvisions are the provisions whose provider keeps its consumers' data (novox/hq ADR 0214):
|
|
||||||
// a module providing one must say how to back it up, or the data the mesh hands out is the data it
|
|
||||||
// cannot restore — issue 241's seven databases. A provision that holds nothing worth keeping (a
|
|
||||||
// route, a cache, a name) is not here; adding one is adding a store.
|
|
||||||
var storeProvisions = map[string]bool{
|
|
||||||
"postgres-database": true,
|
|
||||||
"mssql-database": true,
|
|
||||||
"mongodb-database": true,
|
|
||||||
"s3-bucket": true,
|
|
||||||
"influxdb-api": true,
|
|
||||||
"secret": true,
|
|
||||||
}
|
|
||||||
|
|
||||||
// CheckBackup is a store provider that contributes no backup (novox/hq ADR 0214, "How it is
|
|
||||||
// checked").
|
|
||||||
//
|
|
||||||
// **The catalogue check's, not parsing's.** A manifest already registered and running was written
|
|
||||||
// before the rule; refusing it on read would make the controller refuse the very providers whose
|
|
||||||
// data the rule protects. New definitions meet it in the catalogue check, where they are written.
|
|
||||||
func CheckBackup(m Manifest) []string {
|
|
||||||
backs := false
|
|
||||||
for _, c := range m.Contributions {
|
|
||||||
if s, known := SeatNamed(c.Seat); known && s.Name == BackupSeat && c.Kind == "backup" {
|
|
||||||
backs = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if backs {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var problems []string
|
|
||||||
for _, o := range m.Provides {
|
|
||||||
if storeProvisions[o.Name] {
|
|
||||||
problems = append(problems, fmt.Sprintf(
|
|
||||||
"%s provides %s, which keeps its consumers' data, and contributes no backup to %s; a "+
|
|
||||||
"store says how its data is copied (novox/hq ADR 0214)", m.Module, o.Name, BackupSeat))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return problems
|
|
||||||
}
|
|
||||||
|
|
||||||
// SeatContribution is one piece of configuration a module gives a seat's holder to place.
|
// SeatContribution is one piece of configuration a module gives a seat's holder to place.
|
||||||
type SeatContribution struct {
|
type SeatContribution struct {
|
||||||
// Seat is the seat whose holder places it.
|
// Seat is the seat whose holder places it.
|
||||||
@@ -188,7 +148,11 @@ func seatPlaceholderProblems(m Manifest, r map[string]any) []string {
|
|||||||
// comment line naming it in the tool's grammar, and empty when nothing is contributed. A kind that
|
// comment line naming it in the tool's grammar, and empty when nothing is contributed. A kind that
|
||||||
// takes directories has each contributor's `${dir:<id>}` filled with where that module's directory
|
// takes directories has each contributor's `${dir:<id>}` filled with where that module's directory
|
||||||
// is on this machine (novox/hq to-be 43).
|
// is on this machine (novox/hq to-be 43).
|
||||||
func seatContributions(modules []Manifest, seat, kind string, with Rendering) (string, error) {
|
//
|
||||||
|
// A directory placed in a person's home is named `${machine:account-home}/…` (novox/hq to-be 29); in a
|
||||||
|
// kind that takes directories that is filled from the machine's facts as well, so the holder reads a
|
||||||
|
// path, never a placeholder (novox/hq ADR 0233: an item in the operator's home is backed up too).
|
||||||
|
func seatContributions(modules []Manifest, seat, kind string, with Rendering, facts map[string]string) (string, error) {
|
||||||
s, r, ok := receivable(seat, kind)
|
s, r, ok := receivable(seat, kind)
|
||||||
if !ok {
|
if !ok {
|
||||||
return "", nil
|
return "", nil
|
||||||
@@ -197,7 +161,7 @@ func seatContributions(modules []Manifest, seat, kind string, with Rendering) (s
|
|||||||
var b strings.Builder
|
var b strings.Builder
|
||||||
for _, m := range inModuleOrder(modules) {
|
for _, m := range inModuleOrder(modules) {
|
||||||
named := false
|
named := false
|
||||||
for _, c := range m.Contributions {
|
for _, c := range m.allContributions() {
|
||||||
if c.Kind != kind {
|
if c.Kind != kind {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -214,6 +178,27 @@ func seatContributions(modules []Manifest, seat, kind string, with Rendering) (s
|
|||||||
if err != nil && failed == nil {
|
if err != nil && failed == nil {
|
||||||
failed = err
|
failed = err
|
||||||
}
|
}
|
||||||
|
// An operator's path the module was given, as an item of data on it (novox/hq ADR 0233).
|
||||||
|
if accessRef.MatchString(filled) {
|
||||||
|
_, byID, err := accessesFor(m, with.Settings[m.Module])
|
||||||
|
if err == nil {
|
||||||
|
filled, err = accessFill(filled, byID, m.Module)
|
||||||
|
}
|
||||||
|
if err != nil && failed == nil {
|
||||||
|
failed = err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, key := range machineUsed(filled) {
|
||||||
|
value, has := facts[key]
|
||||||
|
if !has {
|
||||||
|
if failed == nil {
|
||||||
|
failed = fmt.Errorf("%s's %s for %s says ${machine:%s}, and this machine says %s",
|
||||||
|
m.Module, kind, s.Name, key, orNothing(namesOfFacts(facts)))
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
filled = strings.ReplaceAll(filled, "${machine:"+key+"}", value)
|
||||||
|
}
|
||||||
content = filled
|
content = filled
|
||||||
}
|
}
|
||||||
b.WriteString(content)
|
b.WriteString(content)
|
||||||
|
|||||||
@@ -123,6 +123,12 @@ func contributedTo(m Manifest) []string {
|
|||||||
out = append(out, s.Name)
|
out = append(out, s.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// And the backup holder for what is irreplaceable (novox/hq ADR 0233): it backs it up or watches the
|
||||||
|
// array it is on. What is valuable is backed up where a holder is — the standard plan — and makes
|
||||||
|
// no machine refuse it for want of one.
|
||||||
|
if m.NeedsBackupHolder() {
|
||||||
|
out = append(out, BackupSeat)
|
||||||
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -256,8 +256,10 @@ var defaultSeats = append([]Seat{
|
|||||||
// disasters. A module contributes `backup` lines — what to run to take a consistent copy, and
|
// disasters. A module contributes `backup` lines — what to run to take a consistent copy, and
|
||||||
// which of its directories to keep.
|
// which of its directories to keep.
|
||||||
{Name: BackupSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0214",
|
{Name: BackupSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0214",
|
||||||
Serves: backupVerbs(),
|
Serves: backupVerbs(),
|
||||||
Receives: []Receivable{{Kind: "backup", Comment: "#", Dirs: true}}},
|
// `backup` is what to run and which paths to keep; `data` every item a module declares, with its
|
||||||
|
// class, for the holder to measure (novox/hq ADR 0233). Both derived from modules' data sections.
|
||||||
|
Receives: []Receivable{{Kind: BackupKindBackup, Comment: "#", Dirs: true}, {Kind: BackupKindData, Comment: "#", Dirs: true}}},
|
||||||
// Deferred (novox/hq ADR 0121): renaming to mesh-private-network is a scope + server/client
|
// Deferred (novox/hq ADR 0121): renaming to mesh-private-network is a scope + server/client
|
||||||
// model change, not a rename, so it stays until that is built.
|
// model change, not a rename, so it stays until that is built.
|
||||||
{Name: "the-private-network", Scope: ScopeNode, Decision: "novox/hq ADR 0110"},
|
{Name: "the-private-network", Scope: ScopeNode, Decision: "novox/hq ADR 0110"},
|
||||||
|
|||||||
@@ -282,8 +282,10 @@ var ControllerVerbs = []Verb{
|
|||||||
"cause": "with answer: the cause in a word (retire-waiting when absent)",
|
"cause": "with answer: the cause in a word (retire-waiting when absent)",
|
||||||
}, nil)},
|
}, nil)},
|
||||||
{Name: "cleanup", Description: "Every consumer a provider holds retired — its age, its size where the " +
|
{Name: "cleanup", Description: "Every consumer a provider holds retired — its age, its size where the " +
|
||||||
"backend knows, and why it was retired. With consumer (and node, module): the provider deletes that one " +
|
"backend knows, and why it was retired — and every module's own data retired on its machine (ADR 0233). " +
|
||||||
"retired consumer — never an active one. With older-than: every retired consumer older than that many " +
|
"With consumer (and node, module): the provider deletes that one retired consumer — never an active one; " +
|
||||||
|
"for a module's own retired item, consumer names the item and the machine's backup holder takes a last " +
|
||||||
|
"restore point of it, then deletes it. With older-than: every retired consumer older than that many " +
|
||||||
"days, listed; deleted only with confirm. Deleting is a hand act, which says why (novox/hq ADR 0230).",
|
"days, listed; deleted only with confirm. Deleting is a hand act, which says why (novox/hq ADR 0230).",
|
||||||
Input: schema(map[string]string{
|
Input: schema(map[string]string{
|
||||||
"node": "with consumer: the machine the provider runs on",
|
"node": "with consumer: the machine the provider runs on",
|
||||||
@@ -294,6 +296,15 @@ var ControllerVerbs = []Verb{
|
|||||||
"why": "with consumer or older-than: why — required, and recorded in the hand-act log",
|
"why": "with consumer or older-than: why — required, and recorded in the hand-act log",
|
||||||
"cause": "with consumer or older-than: the cause in a word (cleanup-waiting when absent)",
|
"cause": "with consumer or older-than: the cause in a word (cleanup-waiting when absent)",
|
||||||
}, nil, "confirm")},
|
}, nil, "confirm")},
|
||||||
|
// The data every machine declares (novox/hq ADR 0233).
|
||||||
|
{Name: "data", Description: "Every item of data every machine declares, as the self-check last measured it: " +
|
||||||
|
"its class (irreplaceable, rebuildable, cache), where it is, its size, its newest write, its newest good " +
|
||||||
|
"backup and the bound on it — and what is retired: kept after its module left the machine, removed only by " +
|
||||||
|
"`cleanup delete` (novox/hq ADR 0233).",
|
||||||
|
Input: schema(map[string]string{
|
||||||
|
"machine": "one machine (optional)",
|
||||||
|
"retired": "\"true\": only what is retired",
|
||||||
|
}, nil, "retired")},
|
||||||
{Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " +
|
{Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " +
|
||||||
"`builds` with that id follows it line by line, and the module is registered when the outcome comes.",
|
"`builds` with that id follows it line by line, and the module is registered when the outcome comes.",
|
||||||
Input: schema(map[string]string{
|
Input: schema(map[string]string{
|
||||||
|
|||||||
@@ -0,0 +1,316 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The data a module declares, as the mesh found it on each machine (novox/hq ADR 0233).
|
||||||
|
//
|
||||||
|
// The self-check composes what every machine declares and asks its backup holder what it measured;
|
||||||
|
// this keeps both. An irreplaceable item a machine no longer declares — its module unassigned, or no
|
||||||
|
// longer pulled in — is retired here, not forgotten: kept, with when and why, until a person deletes
|
||||||
|
// it through `cleanup delete`.
|
||||||
|
|
||||||
|
// DeclaredData is one item a machine's composition declares now.
|
||||||
|
type DeclaredData struct {
|
||||||
|
Module, Item, Class string
|
||||||
|
// Owned is whether it is in the module's own directory: only that is the mesh's to retire.
|
||||||
|
Owned bool
|
||||||
|
// Protection is backup, redundancy, both ("backup+redundancy") or none.
|
||||||
|
Protection string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Redundancy is the redundant storage an item is on, as its machine's holder read it.
|
||||||
|
type Redundancy struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Where string `json:"where"`
|
||||||
|
Healthy *bool `json:"healthy"`
|
||||||
|
Said string `json:"said"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Measurement is what a machine's backup holder said of one item. Nil fields were not measured.
|
||||||
|
type Measurement struct {
|
||||||
|
Path string
|
||||||
|
Size *int64
|
||||||
|
LastWrite *time.Time
|
||||||
|
MeasuredAt *time.Time
|
||||||
|
LastBackup *time.Time
|
||||||
|
// Error is what the holder could not measure, when it could not.
|
||||||
|
Error string
|
||||||
|
// Precision is what the size is, as the holder said it: "exact", "dataset …", "partial: …", "no size: …".
|
||||||
|
Precision string
|
||||||
|
Redundancy *Redundancy
|
||||||
|
}
|
||||||
|
|
||||||
|
// DataRecord is one item as the mesh keeps it.
|
||||||
|
type DataRecord struct {
|
||||||
|
Machine, Module, Item, Class, Path string
|
||||||
|
Owned bool
|
||||||
|
Protection string
|
||||||
|
FirstSeen, DeclaredAt time.Time
|
||||||
|
MeasureError string
|
||||||
|
Precision string
|
||||||
|
Redundancy *Redundancy
|
||||||
|
Size *int64
|
||||||
|
LastWrite, MeasuredAt, LastBackup *time.Time
|
||||||
|
RetiredAt *time.Time
|
||||||
|
RetiredWhy string
|
||||||
|
DeletedAt *time.Time
|
||||||
|
DeletedBy, DeletedWhy string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Comparable is whether a size is fit to compare with another: exact, or a dataset's own counters.
|
||||||
|
func Comparable(precision string) bool {
|
||||||
|
return precision == "" || precision == "exact" || strings.HasPrefix(precision, "dataset ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dataset is the ZFS dataset a size was read from, when it was: what several items on one dataset share.
|
||||||
|
func Dataset(precision string) string {
|
||||||
|
rest, ok := strings.CutPrefix(precision, "dataset ")
|
||||||
|
if !ok {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
name, _, _ := strings.Cut(rest, ":")
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retired is whether the item is retired and not deleted.
|
||||||
|
func (r DataRecord) Retired() bool { return r.RetiredAt != nil && r.DeletedAt == nil }
|
||||||
|
|
||||||
|
// Key names it in one string, the way conditions and verbs do: <machine>/<module>/<item>.
|
||||||
|
func (r DataRecord) Key() string { return r.Machine + "/" + r.Module + "/" + r.Item }
|
||||||
|
|
||||||
|
// DataChange is what recording a machine's data changed: what it retired and what came back.
|
||||||
|
type DataChange struct {
|
||||||
|
Retired, Reenabled []DataRecord
|
||||||
|
}
|
||||||
|
|
||||||
|
// readingEvery is how often a measurement is kept as a reading: the shrink is read over days, and a
|
||||||
|
// row every five minutes would say the same thing sixty times an hour.
|
||||||
|
const readingEvery = 55 * time.Minute
|
||||||
|
|
||||||
|
// readingsKept is how long readings are kept.
|
||||||
|
const readingsKept = 90 * 24 * time.Hour
|
||||||
|
|
||||||
|
// dataKey is one item's identity on one machine.
|
||||||
|
type dataKey struct{ module, item string }
|
||||||
|
|
||||||
|
// RecordData keeps what one machine declares now and what its holder measured, at now.
|
||||||
|
//
|
||||||
|
// **Only from a composition that worked.** The caller passes the declared set of a machine whose
|
||||||
|
// composition succeeded; an item missing from it is then really no longer declared. An irreplaceable
|
||||||
|
// one is retired — never deleted, never forgotten — and anything else is forgotten, because what is
|
||||||
|
// rebuildable or a cache is not the mesh's to keep track of once its module is gone. An item declared
|
||||||
|
// again comes back from retirement as it was.
|
||||||
|
func (i *Inventory) RecordData(ctx context.Context, machine string, declared []DeclaredData,
|
||||||
|
measured map[string]map[string]Measurement, why string, now time.Time) (DataChange, error) {
|
||||||
|
var change DataChange
|
||||||
|
tx, err := i.store.Pool().Begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return change, err
|
||||||
|
}
|
||||||
|
defer tx.Rollback(ctx) //nolint:errcheck
|
||||||
|
|
||||||
|
existing := map[dataKey]DataRecord{}
|
||||||
|
rows, err := tx.Query(ctx, dataSelect+` where machine = $1`, machine)
|
||||||
|
if err != nil {
|
||||||
|
return change, err
|
||||||
|
}
|
||||||
|
for rows.Next() {
|
||||||
|
r, err := scanData(rows)
|
||||||
|
if err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return change, err
|
||||||
|
}
|
||||||
|
existing[dataKey{r.Module, r.Item}] = r
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return change, err
|
||||||
|
}
|
||||||
|
|
||||||
|
seen := map[dataKey]bool{}
|
||||||
|
for _, d := range declared {
|
||||||
|
k := dataKey{d.Module, d.Item}
|
||||||
|
seen[k] = true
|
||||||
|
m := measured[d.Module][d.Item]
|
||||||
|
var red struct {
|
||||||
|
Kind, Where, Said *string
|
||||||
|
Healthy *bool
|
||||||
|
}
|
||||||
|
if r := m.Redundancy; r != nil {
|
||||||
|
red.Kind, red.Where, red.Said, red.Healthy = &r.Kind, &r.Where, &r.Said, r.Healthy
|
||||||
|
}
|
||||||
|
was, had := existing[k]
|
||||||
|
if had && was.Retired() {
|
||||||
|
change.Reenabled = append(change.Reenabled, was)
|
||||||
|
}
|
||||||
|
// Deleted and declared again is new data: it starts over.
|
||||||
|
fresh := !had || was.DeletedAt != nil
|
||||||
|
if _, err := tx.Exec(ctx, `
|
||||||
|
insert into data_item (machine, module, item, class, path, first_seen, declared_at,
|
||||||
|
size_bytes, last_write, measured_at, last_backup, owned, protection,
|
||||||
|
measure_error, redundancy_kind, redundancy_where, redundancy_healthy, redundancy_said,
|
||||||
|
precision)
|
||||||
|
values ($1, $2, $3, $4, $5, $6, $6, $7, $8, $9, $10, $12, $13, $14, $15, $16, $17, $18, $19)
|
||||||
|
on conflict (machine, module, item) do update set
|
||||||
|
class = excluded.class,
|
||||||
|
owned = excluded.owned,
|
||||||
|
protection = excluded.protection,
|
||||||
|
precision = case when excluded.measured_at is not null then excluded.precision else data_item.precision end,
|
||||||
|
size_bytes = case when excluded.measured_at is not null then excluded.size_bytes else data_item.size_bytes end,
|
||||||
|
measure_error = case when excluded.measured_at is not null then excluded.measure_error else data_item.measure_error end,
|
||||||
|
redundancy_kind = case when excluded.measured_at is not null then excluded.redundancy_kind else data_item.redundancy_kind end,
|
||||||
|
redundancy_where = case when excluded.measured_at is not null then excluded.redundancy_where else data_item.redundancy_where end,
|
||||||
|
redundancy_healthy = case when excluded.measured_at is not null then excluded.redundancy_healthy else data_item.redundancy_healthy end,
|
||||||
|
redundancy_said = case when excluded.measured_at is not null then excluded.redundancy_said else data_item.redundancy_said end,
|
||||||
|
path = case when excluded.path <> '' then excluded.path else data_item.path end,
|
||||||
|
first_seen = case when $11::boolean then excluded.first_seen else data_item.first_seen end,
|
||||||
|
declared_at = excluded.declared_at,
|
||||||
|
last_write = coalesce(excluded.last_write, data_item.last_write),
|
||||||
|
measured_at = coalesce(excluded.measured_at, data_item.measured_at),
|
||||||
|
last_backup = coalesce(excluded.last_backup, data_item.last_backup),
|
||||||
|
retired_at = null, retired_why = null,
|
||||||
|
deleted_at = null, deleted_by = null, deleted_why = null`,
|
||||||
|
machine, d.Module, d.Item, d.Class, m.Path, now, m.Size, m.LastWrite, m.MeasuredAt, m.LastBackup,
|
||||||
|
fresh, d.Owned, d.Protection, nullable(m.Error), red.Kind, red.Where, red.Healthy, red.Said,
|
||||||
|
nullable(m.Precision)); err != nil {
|
||||||
|
return change, err
|
||||||
|
}
|
||||||
|
if m.Size != nil && m.MeasuredAt != nil && Comparable(m.Precision) {
|
||||||
|
if _, err := tx.Exec(ctx, `
|
||||||
|
insert into data_reading (machine, module, item, at, size_bytes, last_write)
|
||||||
|
select $1, $2, $3, $4, $5, $6
|
||||||
|
where not exists (select 1 from data_reading
|
||||||
|
where machine = $1 and module = $2 and item = $3 and at > $4::timestamptz - $7::interval)`,
|
||||||
|
machine, d.Module, d.Item, *m.MeasuredAt, *m.Size, m.LastWrite,
|
||||||
|
fmt.Sprintf("%d seconds", int(readingEvery.Seconds()))); err != nil {
|
||||||
|
return change, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for k, r := range existing {
|
||||||
|
if seen[k] || r.DeletedAt != nil || r.Retired() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !catalogue.Retires(r.Class) || !r.Owned {
|
||||||
|
if _, err := tx.Exec(ctx, `delete from data_item where machine = $1 and module = $2 and item = $3`,
|
||||||
|
machine, k.module, k.item); err != nil {
|
||||||
|
return change, err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(ctx, `update data_item set retired_at = $4, retired_why = $5
|
||||||
|
where machine = $1 and module = $2 and item = $3`, machine, k.module, k.item, now, why); err != nil {
|
||||||
|
return change, err
|
||||||
|
}
|
||||||
|
at := now
|
||||||
|
r.RetiredAt, r.RetiredWhy = &at, why
|
||||||
|
change.Retired = append(change.Retired, r)
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(ctx, `delete from data_reading where at < $1`, now.Add(-readingsKept)); err != nil {
|
||||||
|
return change, err
|
||||||
|
}
|
||||||
|
return change, tx.Commit(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
const dataSelect = `select machine, module, item, class, path, first_seen, declared_at, size_bytes, last_write,
|
||||||
|
measured_at, last_backup, retired_at, coalesce(retired_why, ''), deleted_at, coalesce(deleted_by, ''),
|
||||||
|
coalesce(deleted_why, ''), owned, protection, coalesce(measure_error, ''), redundancy_kind, redundancy_where,
|
||||||
|
redundancy_healthy, redundancy_said, coalesce(precision, '') from data_item`
|
||||||
|
|
||||||
|
func scanData(rows pgx.Row) (DataRecord, error) {
|
||||||
|
var r DataRecord
|
||||||
|
var kind, where, said *string
|
||||||
|
var healthy *bool
|
||||||
|
err := rows.Scan(&r.Machine, &r.Module, &r.Item, &r.Class, &r.Path, &r.FirstSeen, &r.DeclaredAt, &r.Size,
|
||||||
|
&r.LastWrite, &r.MeasuredAt, &r.LastBackup, &r.RetiredAt, &r.RetiredWhy, &r.DeletedAt, &r.DeletedBy,
|
||||||
|
&r.DeletedWhy, &r.Owned, &r.Protection, &r.MeasureError, &kind, &where, &healthy, &said, &r.Precision)
|
||||||
|
if err == nil && kind != nil {
|
||||||
|
r.Redundancy = &Redundancy{Kind: *kind, Healthy: healthy}
|
||||||
|
if where != nil {
|
||||||
|
r.Redundancy.Where = *where
|
||||||
|
}
|
||||||
|
if said != nil {
|
||||||
|
r.Redundancy.Said = *said
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return r, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func nullable(s string) *string {
|
||||||
|
if s == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &s
|
||||||
|
}
|
||||||
|
|
||||||
|
// Data is every item the mesh keeps, by machine, module and item.
|
||||||
|
func (i *Inventory) Data(ctx context.Context) ([]DataRecord, error) {
|
||||||
|
rows, err := i.store.Pool().Query(ctx, dataSelect+` order by machine, module, item`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var out []DataRecord
|
||||||
|
for rows.Next() {
|
||||||
|
r, err := scanData(rows)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, r)
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// DataOf is one item.
|
||||||
|
func (i *Inventory) DataOf(ctx context.Context, machine, module, item string) (DataRecord, error) {
|
||||||
|
r, err := scanData(i.store.Pool().QueryRow(ctx, dataSelect+` where machine = $1 and module = $2 and item = $3`,
|
||||||
|
machine, module, item))
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return r, fmt.Errorf("the mesh knows no data %s of %s on %s", item, module, machine)
|
||||||
|
}
|
||||||
|
return r, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// DataPeaks is each item's largest reading since a moment, keyed by DataRecord.Key.
|
||||||
|
func (i *Inventory) DataPeaks(ctx context.Context, since time.Time) (map[string]int64, error) {
|
||||||
|
rows, err := i.store.Pool().Query(ctx, `select machine, module, item, max(size_bytes) from data_reading
|
||||||
|
where at >= $1 group by machine, module, item`, since)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
out := map[string]int64{}
|
||||||
|
for rows.Next() {
|
||||||
|
var machine, module, item string
|
||||||
|
var peak int64
|
||||||
|
if err := rows.Scan(&machine, &module, &item, &peak); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out[machine+"/"+module+"/"+item] = peak
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarkDataDeleted records that a person deleted a retired item. Refused for an item not retired.
|
||||||
|
func (i *Inventory) MarkDataDeleted(ctx context.Context, machine, module, item, by, why string, at time.Time) error {
|
||||||
|
tag, err := i.store.Pool().Exec(ctx, `update data_item set deleted_at = $4, deleted_by = $5, deleted_why = $6
|
||||||
|
where machine = $1 and module = $2 and item = $3 and retired_at is not null and deleted_at is null`,
|
||||||
|
machine, module, item, at, by, why)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if tag.RowsAffected() == 0 {
|
||||||
|
return fmt.Errorf("%s of %s on %s is not retired: only retired data is deleted", item, module, machine)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,130 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func size(n int64) *int64 { return &n }
|
||||||
|
|
||||||
|
func at(t time.Time) *time.Time { return &t }
|
||||||
|
|
||||||
|
// What a machine declares is kept with what its holder measured; an irreplaceable item it no longer
|
||||||
|
// declares — its module unassigned — is RETIRED and kept, never forgotten, and comes back as it was when
|
||||||
|
// declared again; anything else no longer declared is forgotten (novox/hq ADR 0233).
|
||||||
|
func TestAnUndeclaredIrreplaceableItemIsRetiredNotForgotten(t *testing.T) {
|
||||||
|
inv := fresh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
||||||
|
declared := []DeclaredData{
|
||||||
|
{Module: "home-assistant", Item: "config", Class: "irreplaceable", Owned: true},
|
||||||
|
{Module: "searxng", Item: "valkey", Class: "cache"},
|
||||||
|
{Module: "ollama", Item: "models", Class: "rebuildable"},
|
||||||
|
}
|
||||||
|
measured := map[string]map[string]Measurement{"home-assistant": {"config": {Path: "/var/lib/home-assistant/config",
|
||||||
|
Size: size(900 << 20), LastWrite: at(now.Add(-time.Minute)), MeasuredAt: at(now), LastBackup: at(now.Add(-6 * time.Hour))}}}
|
||||||
|
if _, err := inv.RecordData(ctx, "home", declared, measured, "", now); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
all, err := inv.Data(ctx)
|
||||||
|
if err != nil || len(all) != 3 {
|
||||||
|
t.Fatalf("%+v, %v", all, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unassigned: nothing of it is declared any more.
|
||||||
|
later := now.Add(time.Hour)
|
||||||
|
change, err := inv.RecordData(ctx, "home", nil, nil, "home-assistant unassigned", later)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(change.Retired) != 1 || change.Retired[0].Item != "config" {
|
||||||
|
t.Fatalf("retired %+v", change.Retired)
|
||||||
|
}
|
||||||
|
all, err = inv.Data(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(all) != 1 || !all[0].Retired() || all[0].Path != "/var/lib/home-assistant/config" ||
|
||||||
|
*all[0].Size != 900<<20 || all[0].RetiredWhy != "home-assistant unassigned" {
|
||||||
|
t.Fatalf("the irreplaceable item is not kept retired with where it is: %+v", all)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A second run that still does not declare it changes nothing: retired once, not again.
|
||||||
|
change, err = inv.RecordData(ctx, "home", nil, nil, "again", later.Add(time.Hour))
|
||||||
|
if err != nil || len(change.Retired) != 0 {
|
||||||
|
t.Fatalf("retired twice: %+v, %v", change, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deleting needs it retired; assigned again it is not retired any more, and its history stays.
|
||||||
|
if err := inv.MarkDataDeleted(ctx, "home", "searxng", "valkey", "p", "w", later); err == nil {
|
||||||
|
t.Fatal("deleting something the mesh does not hold retired was recorded")
|
||||||
|
}
|
||||||
|
change, err = inv.RecordData(ctx, "home", declared[:1], nil, "", later.Add(2*time.Hour))
|
||||||
|
if err != nil || len(change.Reenabled) != 1 {
|
||||||
|
t.Fatalf("declared again: %+v, %v", change, err)
|
||||||
|
}
|
||||||
|
r, err := inv.DataOf(ctx, "home", "home-assistant", "config")
|
||||||
|
if err != nil || r.Retired() || !r.FirstSeen.Equal(now) || r.Size == nil {
|
||||||
|
t.Fatalf("declared again lost what was known: %+v, %v", r, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retired and then deleted by a person: recorded, never by dropping the row.
|
||||||
|
if _, err := inv.RecordData(ctx, "home", nil, nil, "unassigned again", later.Add(3*time.Hour)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.MarkDataDeleted(ctx, "home", "home-assistant", "config", "jochen", "moved to the anchor", later.Add(4*time.Hour)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
r, err = inv.DataOf(ctx, "home", "home-assistant", "config")
|
||||||
|
if err != nil || r.DeletedAt == nil || r.DeletedBy != "jochen" || r.Retired() {
|
||||||
|
t.Fatalf("a deletion is not on record: %+v, %v", r, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A reading is kept at most once an hour, and the peak is read over the window asked.
|
||||||
|
func TestReadingsAreHourlyAndThePeakIsTheLargest(t *testing.T) {
|
||||||
|
inv := fresh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
now := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)
|
||||||
|
declared := []DeclaredData{{Module: "grafana", Item: "data", Class: "valuable", Owned: true}}
|
||||||
|
for i, s := range []int64{100, 120, 999, 130, 40} {
|
||||||
|
when := now.Add(time.Duration(i) * 20 * time.Minute)
|
||||||
|
if _, err := inv.RecordData(ctx, "ace", declared, map[string]map[string]Measurement{"grafana": {"data": {
|
||||||
|
Path: "/var/lib/grafana/data", Size: size(s), MeasuredAt: at(when)}}}, "", when); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var n int
|
||||||
|
if err := inv.store.Pool().QueryRow(ctx, `select count(*) from data_reading`).Scan(&n); err != nil || n != 2 {
|
||||||
|
t.Fatalf("%d readings kept for five measurements over 80 minutes, want 2 (one an hour): %v", n, err)
|
||||||
|
}
|
||||||
|
peaks, err := inv.DataPeaks(ctx, now.Add(-time.Hour))
|
||||||
|
if err != nil || peaks["ace/grafana/data"] != 130 {
|
||||||
|
t.Fatalf("peak %v, %v", peaks, err)
|
||||||
|
}
|
||||||
|
r, _ := inv.DataOf(ctx, "ace", "grafana", "data")
|
||||||
|
if r.Size == nil || *r.Size != 40 {
|
||||||
|
t.Fatalf("the newest measurement is not the one on record: %+v", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A partial walk's lower bound is kept as the newest measurement, said as partial, and never kept as a
|
||||||
|
// reading a shrink would be read against.
|
||||||
|
func TestAPartialMeasurementIsNeverAReading(t *testing.T) {
|
||||||
|
inv := fresh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
now := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)
|
||||||
|
declared := []DeclaredData{{Module: "big", Item: "data", Class: "valuable", Owned: true}}
|
||||||
|
if _, err := inv.RecordData(ctx, "home", declared, map[string]map[string]Measurement{"big": {"data": {
|
||||||
|
Path: "/srv/big", Size: size(5), MeasuredAt: at(now), Precision: "partial: stopped"}}}, "", now); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var n int
|
||||||
|
if err := inv.store.Pool().QueryRow(ctx, `select count(*) from data_reading`).Scan(&n); err != nil || n != 0 {
|
||||||
|
t.Fatalf("%d readings from a partial measurement: %v", n, err)
|
||||||
|
}
|
||||||
|
r, err := inv.DataOf(ctx, "home", "big", "data")
|
||||||
|
if err != nil || r.Precision != "partial: stopped" || Comparable(r.Precision) {
|
||||||
|
t.Fatalf("%+v, %v", r, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
-- The data a module declares, as the mesh found it on each machine (novox/hq ADR 0233).
|
||||||
|
--
|
||||||
|
-- A module's manifest says what data it keeps and of which class. The self-check asks each machine's
|
||||||
|
-- backup holder what it measured of every declared item — its size, its last write, its last good
|
||||||
|
-- backup — and keeps that here: so a shrink is read against what the item held before, an item a
|
||||||
|
-- machine no longer declares is still known to be there, and an empty copy of an item is told from a
|
||||||
|
-- full one somewhere else.
|
||||||
|
--
|
||||||
|
-- **Keyed by the machine's name, not a reference.** The data outlives the machine record, as a
|
||||||
|
-- binding's provider does (migration 0071): a machine leaving the mesh must not turn the record of
|
||||||
|
-- what it holds into nothing.
|
||||||
|
--
|
||||||
|
-- **Retired, never removed.** An irreplaceable or valuable item in a module's own directory that its
|
||||||
|
-- machine no longer declares — its module unassigned — is marked retired, with when and why, and stays
|
||||||
|
-- until a person deletes it through `cleanup delete` (ADR 0230); the deletion is recorded here too,
|
||||||
|
-- never by dropping the row. An item on an operator's path (an access) is never the mesh's to retire.
|
||||||
|
--
|
||||||
|
-- Numbered 0072, past 0071, the highest on main or any open branch when this was written.
|
||||||
|
create table data_item (
|
||||||
|
machine text not null,
|
||||||
|
module text not null,
|
||||||
|
item text not null,
|
||||||
|
class text not null,
|
||||||
|
-- Whether it is in the module's own directory (the mesh's to retire) or an operator's path, and
|
||||||
|
-- how it is protected: backup, redundancy, both, or none.
|
||||||
|
owned boolean not null default true,
|
||||||
|
protection text not null default '',
|
||||||
|
-- Where it is on the machine, as the backup holder last said; empty until one has.
|
||||||
|
path text not null default '',
|
||||||
|
first_seen timestamptz not null default now(),
|
||||||
|
-- When a composition of the machine last declared it.
|
||||||
|
declared_at timestamptz not null default now(),
|
||||||
|
-- The newest measurement: size in bytes, the newest write inside it, and when it was measured.
|
||||||
|
size_bytes bigint,
|
||||||
|
last_write timestamptz,
|
||||||
|
measured_at timestamptz,
|
||||||
|
-- The newest good backup that covers it.
|
||||||
|
last_backup timestamptz,
|
||||||
|
-- What the holder could not measure, when it could not: the path gone, a walk refused.
|
||||||
|
measure_error text,
|
||||||
|
-- What the newest size is: exact, a dataset's whole size, partial (a lower bound) or none.
|
||||||
|
precision text,
|
||||||
|
-- The redundant storage it is on, as the holder read it: zfs, md or btrfs, which pool or device,
|
||||||
|
-- whether it is healthy, and what it said.
|
||||||
|
redundancy_kind text,
|
||||||
|
redundancy_where text,
|
||||||
|
redundancy_healthy boolean,
|
||||||
|
redundancy_said text,
|
||||||
|
retired_at timestamptz,
|
||||||
|
retired_why text,
|
||||||
|
deleted_at timestamptz,
|
||||||
|
deleted_by text,
|
||||||
|
deleted_why text,
|
||||||
|
primary key (machine, module, item)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- One row per measurement kept, at most one an hour per item, for ninety days: what a shrink is
|
||||||
|
-- read against. Only a comparable size is kept: exact, or a dataset's own counters — never a partial
|
||||||
|
-- walk's lower bound.
|
||||||
|
create table data_reading (
|
||||||
|
machine text not null,
|
||||||
|
module text not null,
|
||||||
|
item text not null,
|
||||||
|
at timestamptz not null,
|
||||||
|
size_bytes bigint not null,
|
||||||
|
last_write timestamptz,
|
||||||
|
primary key (machine, module, item, at)
|
||||||
|
);
|
||||||
@@ -233,10 +233,13 @@ type RetirementRejected struct {
|
|||||||
|
|
||||||
// RetirementState is a provider's answer to provisioner_retirement.
|
// RetirementState is a provider's answer to provisioner_retirement.
|
||||||
type RetirementState struct {
|
type RetirementState struct {
|
||||||
Resource string `json:"resource"`
|
Resource string `json:"resource"`
|
||||||
Node string `json:"node"`
|
Node string `json:"node"`
|
||||||
Held []string `json:"held"`
|
Held []string `json:"held"`
|
||||||
StablePasses int `json:"stable_passes"`
|
// HeldSizes is what each held consumer keeps on the backend, in bytes, where the backend can say
|
||||||
|
// (novox/hq ADR 0233): what an empty replacement of a consumer's data is told by.
|
||||||
|
HeldSizes map[string]int64 `json:"held_sizes,omitempty"`
|
||||||
|
StablePasses int `json:"stable_passes"`
|
||||||
// StableForSeconds is how long the same unasked set must hold as well (ADR 0230: ten minutes).
|
// StableForSeconds is how long the same unasked set must hold as well (ADR 0230: ten minutes).
|
||||||
StableForSeconds int `json:"stable_for_seconds,omitempty"`
|
StableForSeconds int `json:"stable_for_seconds,omitempty"`
|
||||||
// RetiresBy is "disable", or "mark-only" for a provider that cannot disable a consumer.
|
// RetiresBy is "disable", or "mark-only" for a provider that cannot disable a consumer.
|
||||||
|
|||||||
Reference in New Issue
Block a user