Derive data protection from a module's declared data (hq ADR 0233)
A module's data section says what it keeps and how precious it is; the backup holder's lines, binding stickiness, retirement on unassign and D13's conditions follow from it, so issue 273's empty replacement is said and an unassigned module's data is remembered, not forgotten.
This commit is contained in:
jochen
@@ -224,6 +224,10 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
||||
for _, line := range unheldChange(shelf, node, assigned, left) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233).
|
||||
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
return answer + blockedElsewhere(ctx, open, node), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -93,6 +93,15 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
}
|
||||
failed += len(identities)
|
||||
|
||||
// And the data each module keeps (novox/hq ADR 0233): a provider that grants says what it keeps for
|
||||
// its consumers, a directory a container writes is declared, and no backup line is written by hand.
|
||||
data := catalogue.DataProblems(shelf)
|
||||
sort.Strings(data)
|
||||
for _, p := range data {
|
||||
fmt.Fprintln(out, p)
|
||||
}
|
||||
failed += len(data)
|
||||
|
||||
var names []string
|
||||
for name := range shelf {
|
||||
names = append(names, name)
|
||||
@@ -121,6 +130,14 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
if len(m.Reads) > 0 {
|
||||
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
||||
}
|
||||
// The data it keeps, by class, so a reviewer sees what the mesh will protect and how.
|
||||
if items := m.DataItems(); len(items) > 0 {
|
||||
kept := make([]string, 0, len(items))
|
||||
for _, it := range items {
|
||||
kept = append(kept, it.ID+" ("+it.Class+")")
|
||||
}
|
||||
fmt.Fprintf(out, ", keeps %s", strings.Join(kept, ", "))
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
}
|
||||
if failed > 0 {
|
||||
|
||||
@@ -0,0 +1,883 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A module declares the data it holds, and the mesh protects and watches it from that declaration
|
||||
// (novox/hq ADR 0233).
|
||||
//
|
||||
// The self-check's D13 composes what every machine declares, asks each machine's backup holder what
|
||||
// it measured of every item — size, newest write, newest good backup, the redundant storage it is on —
|
||||
// and keeps both. From that, and from what every provider says it holds for its consumers, it raises,
|
||||
// each URGENT for what is irreplaceable and a WARNING for what is valuable (the operator's ranking):
|
||||
//
|
||||
// - `data-shrank`: an item holds less than half of its largest size in seven days, and at least
|
||||
// shrinkFloor less; `data-missing`: its path is gone;
|
||||
// - `empty-replacement`: an item, or a consumer's data at a provider, is less than half the size of a
|
||||
// copy of the same thing kept elsewhere — on 2026-10-05 five applications ran for twenty hours on
|
||||
// empty databases while their real ones sat on another machine (issue 273);
|
||||
// - `data-held-twice` (warning): a consumer has active data at two providers and their sizes cannot
|
||||
// be compared;
|
||||
// - `data-quiet`: an item said to be written all the time has not been, within its bound;
|
||||
// - `backup-stale`: an item's newest good backup is older than its bound, or there is none;
|
||||
// - `array-degraded`: the redundant storage an item is on is not healthy, or cannot be read;
|
||||
// `protection-missing`: an item said to be protected by redundancy is on storage that is not;
|
||||
// - `cleanup-waiting` (warning): an item retired more than thirty days, waiting for a person.
|
||||
//
|
||||
// And it retires: an irreplaceable or valuable item in a module's own directory that its machine no
|
||||
// longer declares — its module unassigned — is kept, marked retired with when and why, and listed by
|
||||
// `cleanup list` until `cleanup delete` removes it. The node-engine never deletes a directory with
|
||||
// anything in it; this is the record of what it kept. An operator's path is never retired or deleted.
|
||||
|
||||
// The condition kinds of D13.
|
||||
const (
|
||||
kindDataShrank = "data-shrank"
|
||||
kindEmptyReplacement = "empty-replacement"
|
||||
kindDataHeldTwice = "data-held-twice"
|
||||
kindDataQuiet = "data-quiet"
|
||||
kindBackupStale = "backup-stale"
|
||||
kindDataUnmeasured = "data-unmeasured"
|
||||
// kindDataMissing is a watched item whose path is gone.
|
||||
kindDataMissing = "data-missing"
|
||||
// kindArrayDegraded is redundant storage watched data is on that is not healthy, or cannot be read.
|
||||
kindArrayDegraded = "array-degraded"
|
||||
// kindProtectionMissing is an item said to be protected by redundancy, on storage that is not.
|
||||
kindProtectionMissing = "protection-missing"
|
||||
)
|
||||
|
||||
// probeDataID is the self-check's id for this probe.
|
||||
const probeDataID = "D13"
|
||||
|
||||
// The bounds the findings are read against.
|
||||
var (
|
||||
// shrinkWindow is how far back the largest size is looked for.
|
||||
shrinkWindow = 7 * 24 * time.Hour
|
||||
// shrinkFloor is the least loss that is worth saying: two empty databases differ by a few
|
||||
// megabytes, and half of almost nothing is noise.
|
||||
shrinkFloor int64 = 16 << 20
|
||||
// dataAsk is how long one machine's holder, or one provider, is given to answer.
|
||||
dataAsk = 8 * time.Second
|
||||
)
|
||||
|
||||
// keyOfItem is one item's condition id: its machine, module and item.
|
||||
func keyOfItem(machine, module, item string) string { return machine + "." + module + "." + item }
|
||||
|
||||
// holderAnswer is what a node-backup holder's `backed-up` says of one module (ADR 0233 adds Data).
|
||||
type holderAnswer struct {
|
||||
Module string `json:"module"`
|
||||
Data []holderItem `json:"data"`
|
||||
}
|
||||
|
||||
// holderItem is one item as the holder measured it.
|
||||
type holderItem struct {
|
||||
Item string `json:"item"`
|
||||
Class string `json:"class"`
|
||||
Path string `json:"path"`
|
||||
SizeBytes *int64 `json:"size_bytes"`
|
||||
LastWrite *time.Time `json:"last_write"`
|
||||
MeasuredAt *time.Time `json:"measured_at"`
|
||||
LastBackup *time.Time `json:"last_backup"`
|
||||
Error string `json:"error,omitempty"`
|
||||
// Redundancy is the redundant storage the item is on, where the holder could tell (ADR 0233).
|
||||
Redundancy *inventory.Redundancy `json:"redundancy,omitempty"`
|
||||
}
|
||||
|
||||
// readHolder reads a holder's answer into measurements by module and item.
|
||||
func readHolder(raw json.RawMessage) (map[string]map[string]inventory.Measurement, error) {
|
||||
var modules []holderAnswer
|
||||
if err := json.Unmarshal(raw, &modules); err != nil {
|
||||
return nil, fmt.Errorf("its answer is not readable: %w", err)
|
||||
}
|
||||
out := map[string]map[string]inventory.Measurement{}
|
||||
for _, m := range modules {
|
||||
for _, it := range m.Data {
|
||||
if out[m.Module] == nil {
|
||||
out[m.Module] = map[string]inventory.Measurement{}
|
||||
}
|
||||
out[m.Module][it.Item] = inventory.Measurement{Path: it.Path, Size: it.SizeBytes, LastWrite: it.LastWrite,
|
||||
MeasuredAt: it.MeasuredAt, LastBackup: it.LastBackup, Error: it.Error, Redundancy: it.Redundancy}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// declaredOn is every data item a machine's composition declares, and whether something there holds
|
||||
// node-backup to measure them.
|
||||
func declaredOn(plan catalogue.Resolution) ([]inventory.DeclaredData, bool) {
|
||||
var out []inventory.DeclaredData
|
||||
held := false
|
||||
for _, m := range plan.Modules {
|
||||
for _, c := range m.Claims {
|
||||
if s, known := catalogue.SeatNamed(c.Name); known && s.Name == catalogue.BackupSeat {
|
||||
held = true
|
||||
}
|
||||
}
|
||||
for _, it := range m.DataItems() {
|
||||
out = append(out, inventory.DeclaredData{Module: m.Module, Item: it.ID, Class: it.Class,
|
||||
Owned: it.OwnedByModule(), Protection: it.Protection()})
|
||||
}
|
||||
}
|
||||
return out, held
|
||||
}
|
||||
|
||||
// consumerCopy is one provider's account of one consumer: where, how big, and whether still active.
|
||||
type consumerCopy struct {
|
||||
Node, Module, Consumer string
|
||||
Size *int64
|
||||
Retired bool
|
||||
// Class is how precious the consumer's data is: the stricter of what the provider keeps for its
|
||||
// consumers and what the consumer says it keeps there (`kept-by`).
|
||||
Class string
|
||||
}
|
||||
|
||||
// probeData is D13.
|
||||
func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
if d.js == nil {
|
||||
return nil, errors.New("no bus to ask the machines over")
|
||||
}
|
||||
open := d.open
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
now := time.Now()
|
||||
|
||||
type machine struct {
|
||||
name string
|
||||
declared []inventory.DeclaredData
|
||||
held bool
|
||||
measured map[string]map[string]inventory.Measurement
|
||||
askErr error
|
||||
}
|
||||
var machines []*machine
|
||||
for _, n := range nodes {
|
||||
plan, _, err := planFor(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
// A machine that cannot be worked out declares nothing this run — which is not the same as
|
||||
// declaring nothing: retiring its data on that would be acting on an unreadable result.
|
||||
continue
|
||||
}
|
||||
declared, held := declaredOn(plan)
|
||||
machines = append(machines, &machine{name: n.Name, declared: declared, held: held})
|
||||
}
|
||||
// Every holder asked at once, as D8 asks every ban list.
|
||||
var wg sync.WaitGroup
|
||||
for _, m := range machines {
|
||||
if !m.held || !heard[m.name] {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(m *machine) {
|
||||
defer wg.Done()
|
||||
asking, cancel := context.WithTimeout(ctx, dataAsk)
|
||||
defer cancel()
|
||||
raw, err := askSeatTool(asking, d.js.Conn(), catalogue.BackupSeat, "backed-up", m.name)
|
||||
if err == nil {
|
||||
m.measured, err = readHolder(raw)
|
||||
}
|
||||
m.askErr = err
|
||||
}(m)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
var out []conditions.Observation
|
||||
for _, m := range machines {
|
||||
if m.askErr != nil {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, Token: kindDataUnmeasured,
|
||||
Kind: kindDataUnmeasured, Machine: m.name, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s's backup holder did not say what it measured of the data declared there, so "+
|
||||
"nothing about that data is known this run: %s", m.name, firstLine(m.askErr.Error())),
|
||||
Said: firstLine(m.askErr.Error())})
|
||||
}
|
||||
why := fmt.Sprintf("no longer declared on %s: its module was unassigned there, or is no longer pulled in", m.name)
|
||||
change, err := open.inventory.RecordData(ctx, m.name, m.declared, m.measured, why, now)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("what %s holds could not be kept: %w", m.name, err)
|
||||
}
|
||||
for _, r := range change.Retired {
|
||||
log.Printf("data: %s of %s on %s RETIRED, kept at %s: %s — `cleanup list` shows it, and only `cleanup "+
|
||||
"delete` removes it (novox/hq ADR 0233)", r.Item, r.Module, r.Machine, orUnknownPath(r.Path), why)
|
||||
}
|
||||
for _, r := range change.Reenabled {
|
||||
log.Printf("data: %s of %s on %s is declared again, no longer retired", r.Item, r.Module, r.Machine)
|
||||
}
|
||||
}
|
||||
|
||||
records, err := open.inventory.Data(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
peaks, err := open.inventory.DataPeaks(ctx, now.Add(-shrinkWindow))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bindings, err := open.inventory.Bindings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
upgraded, keptBy := keptByClasses(bindings, shelf)
|
||||
copies, err := consumerCopies(ctx, d.js.Conn(), open.inventory, shelf, keptBy)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, dataFindings(records, peaks, shelf, copies, upgraded, now)...)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func orUnknownPath(p string) string {
|
||||
if p == "" {
|
||||
return "a path its backup holder never named"
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// consumerCopies asks every provider of a provision whose consumers' data is kept what it holds, at
|
||||
// once. One that cannot answer is passed over: it says nothing about any copy, which is not a finding.
|
||||
func consumerCopies(ctx context.Context, conn *nats.Conn, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest, keptBy map[string]string) ([]consumerCopy, error) {
|
||||
instances, err := providerInstances(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var asked []providerInstance
|
||||
for _, p := range instances {
|
||||
m := shelf[p.Module]
|
||||
keeps := false
|
||||
for provision := range m.Grants {
|
||||
keeps = keeps || m.KeepsConsumerData(provision)
|
||||
}
|
||||
if keeps {
|
||||
asked = append(asked, p)
|
||||
}
|
||||
}
|
||||
states := make([]*link.RetirementState, len(asked))
|
||||
var wg sync.WaitGroup
|
||||
for i, p := range asked {
|
||||
wg.Add(1)
|
||||
go func(i int, p providerInstance) {
|
||||
defer wg.Done()
|
||||
asking, cancel := context.WithTimeout(ctx, dataAsk)
|
||||
defer cancel()
|
||||
if s, err := askRetirement(asking, conn, p); err == nil {
|
||||
states[i] = &s
|
||||
}
|
||||
}(i, p)
|
||||
}
|
||||
wg.Wait()
|
||||
var out []consumerCopy
|
||||
for i, p := range asked {
|
||||
s := states[i]
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
class := consumersClass(shelf[p.Module])
|
||||
for _, c := range s.Held {
|
||||
cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: c,
|
||||
Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+c])}
|
||||
if size, ok := s.HeldSizes[c]; ok && size >= 0 {
|
||||
size := size
|
||||
cp.Size = &size
|
||||
}
|
||||
out = append(out, cp)
|
||||
}
|
||||
for _, r := range s.Retired {
|
||||
if r.Kind != "" && r.Kind != "consumer" {
|
||||
continue
|
||||
}
|
||||
cp := consumerCopy{Node: p.Node, Module: p.Module, Consumer: r.Consumer, Retired: true,
|
||||
Class: catalogue.StricterClass(class, keptBy[p.Module+"/"+r.Consumer])}
|
||||
if r.SizeBytes != nil && *r.SizeBytes >= 0 {
|
||||
cp.Size = r.SizeBytes
|
||||
}
|
||||
out = append(out, cp)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// severityOf is how loud a finding about data of a class is: urgent for what is irreplaceable, a warning
|
||||
// for anything else watched (the operator's ranking, ADR 0233).
|
||||
func severityOf(class string) conditions.Severity {
|
||||
if class == catalogue.ClassIrreplaceable {
|
||||
return conditions.Urgent
|
||||
}
|
||||
return conditions.Warning
|
||||
}
|
||||
|
||||
// consumersClass is the most precious class a provider keeps any of its consumers' data as.
|
||||
func consumersClass(m catalogue.Manifest) string {
|
||||
class := catalogue.ClassNone
|
||||
for provision := range m.Grants {
|
||||
if c, ok := m.ConsumerDataOf(provision); ok {
|
||||
class = catalogue.StricterClass(class, c.Class)
|
||||
} else if m.KeepsConsumerData(provision) {
|
||||
class = catalogue.StricterClass(class, catalogue.ClassValuable)
|
||||
}
|
||||
}
|
||||
return class
|
||||
}
|
||||
|
||||
// keptByClasses is what consumers say of the data they keep with their providers (`kept-by`), read
|
||||
// through where each is bound: by provider module and consumer identity, the class of that consumer's
|
||||
// data there; and by provider item key (machine/module/item), the class the item holding it is held to.
|
||||
func keptByClasses(bindings []inventory.Binding, shelf map[string]catalogue.Manifest) (map[string]string, map[string]string) {
|
||||
upgraded, keptBy := map[string]string{}, map[string]string{}
|
||||
for _, b := range bindings {
|
||||
m, ok := shelf[b.Consumer]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
k, said := m.KeptByOf(b.Provision)
|
||||
if !said {
|
||||
continue
|
||||
}
|
||||
identity := catalogue.ConsumerIdentity(b.Machine, catalogue.IdentitySource(m.Slug, m.Module))
|
||||
key := b.Provider.Module + "/" + identity
|
||||
keptBy[key] = catalogue.StricterClass(keptBy[key], k.Class)
|
||||
if pc, ok := shelf[b.Provider.Module].ConsumerDataOf(b.Provision); ok && pc.In != "" {
|
||||
if _, own := shelf[b.Provider.Module].DataItem(pc.In); own {
|
||||
item := b.Provider.Node + "/" + b.Provider.Module + "/" + pc.In
|
||||
upgraded[item] = catalogue.StricterClass(upgraded[item], k.Class)
|
||||
}
|
||||
}
|
||||
}
|
||||
return upgraded, keptBy
|
||||
}
|
||||
|
||||
// dataFindings is every condition the data on record raises now. A function of what is known, so the
|
||||
// incident's shape is tested without a mesh. upgraded is the class an item is held to where a consumer
|
||||
// of its module keeps data in it more precious than its own class says (`kept-by`), by its key.
|
||||
func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf map[string]catalogue.Manifest,
|
||||
copies []consumerCopy, upgraded map[string]string, now time.Time) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
byItem := map[string][]inventory.DataRecord{}
|
||||
arrays := map[string][]inventory.DataRecord{}
|
||||
for _, r := range records {
|
||||
if r.DeletedAt != nil {
|
||||
continue
|
||||
}
|
||||
class := catalogue.StricterClass(r.Class, upgraded[r.Key()])
|
||||
r.Class = class
|
||||
byItem[r.Module+"/"+r.Item] = append(byItem[r.Module+"/"+r.Item], r)
|
||||
item, declared := shelf[r.Module].DataItem(r.Item)
|
||||
id := keyOfItem(r.Machine, r.Module, r.Item)
|
||||
if r.Retired() {
|
||||
if now.Sub(*r.RetiredAt) > cleanupAfter {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: "cleanup",
|
||||
Kind: kindCleanupWaiting, Machine: r.Machine, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept at %s since %s; `cleanup "+
|
||||
"delete %s %s %s --why …` once a person has decided, or assign %s there again",
|
||||
r.Item, r.Module, r.Machine, r.Class, sizeWords(r.Size), int(now.Sub(*r.RetiredAt).Hours()/24),
|
||||
orUnknownPath(r.Path), r.RetiredWhy, r.Machine, r.Module, r.Item, r.Module)})
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !catalogue.Watched(class) {
|
||||
continue
|
||||
}
|
||||
severity := severityOf(class)
|
||||
if r.Redundancy != nil {
|
||||
where := r.Machine + "/" + r.Redundancy.Kind + ":" + r.Redundancy.Where
|
||||
arrays[where] = append(arrays[where], r)
|
||||
} else if declared && item.Redundancy != "" && r.MeasuredAt != nil && r.MeasureError == "" {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindProtectionMissing,
|
||||
Kind: kindProtectionMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s (%s) is said to be protected by the redundancy of the storage it is on, "+
|
||||
"and %s is on nothing the backup holder can read as redundant: it has no protection the mesh can see",
|
||||
r.Item, r.Module, r.Machine, class, orUnknownPath(r.Path))})
|
||||
}
|
||||
if r.MeasureError != "" && strings.Contains(r.MeasureError, "does not exist") {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataMissing,
|
||||
Kind: kindDataMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s (%s) is gone: %s does not exist any more", r.Item, r.Module, r.Machine,
|
||||
class, orUnknownPath(r.Path))})
|
||||
} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && *r.Size*2 < peak && peak-*r.Size >= shrinkFloor {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataShrank,
|
||||
Kind: kindDataShrank, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s (%s) shrank to %s from %s within %d days — more than half of what it "+
|
||||
"held is gone. If that was meant, silence this with why; if not, `node-backup.restore` puts the last "+
|
||||
"good copy beside it", r.Item, r.Module, r.Machine, class, sizeWords(r.Size), sizeWords(&peak),
|
||||
int(shrinkWindow.Hours()/24))})
|
||||
}
|
||||
if !declared {
|
||||
continue
|
||||
}
|
||||
if within := item.ActiveWithin(); within > 0 && r.LastWrite != nil && now.Sub(*r.LastWrite) > within {
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataQuiet,
|
||||
Kind: kindDataQuiet, Machine: r.Machine, Severity: severity,
|
||||
Summary: fmt.Sprintf("%s of %s on %s is written all the time, and has not been since %s (its bound is %s): "+
|
||||
"whatever writes it has stopped", r.Item, r.Module, r.Machine, r.LastWrite.UTC().Format(time.RFC3339),
|
||||
within)})
|
||||
}
|
||||
// A backup is required of what is irreplaceable and copied; of what is valuable it is the standard
|
||||
// plan, said only where the machine was measured — where a holder is there to take it.
|
||||
if item.BackedUp() && (class == catalogue.ClassIrreplaceable || r.MeasuredAt != nil) {
|
||||
within := item.BackupWithin()
|
||||
switch {
|
||||
case r.LastBackup == nil && now.Sub(r.FirstSeen) > within:
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale,
|
||||
Kind: kindBackupStale, Machine: r.Machine, Severity: severity,
|
||||
Summary: fmt.Sprintf("%s of %s on %s is %s and has no good backup on record, %s after it was first "+
|
||||
"declared — is node-backup held there, and do its nights succeed? (`node-backup.backed-up`)",
|
||||
r.Item, r.Module, r.Machine, class, now.Sub(r.FirstSeen).Round(time.Hour))})
|
||||
case r.LastBackup != nil && now.Sub(*r.LastBackup) > within:
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindBackupStale,
|
||||
Kind: kindBackupStale, Machine: r.Machine, Severity: severity,
|
||||
Summary: fmt.Sprintf("%s of %s on %s is %s and its newest good backup is from %s, older than its bound "+
|
||||
"of %s", r.Item, r.Module, r.Machine, class, r.LastBackup.UTC().Format(time.RFC3339), within)})
|
||||
}
|
||||
}
|
||||
}
|
||||
// The redundant storage watched data is on: one condition per array, as loud as the most precious
|
||||
// item on it — the array, not each item, is what degrades.
|
||||
for _, where := range keysSorted(arrays) {
|
||||
rs := arrays[where]
|
||||
red := rs[0].Redundancy
|
||||
if red.Healthy != nil && *red.Healthy {
|
||||
continue
|
||||
}
|
||||
class, machine := catalogue.ClassValuable, rs[0].Machine
|
||||
var names []string
|
||||
for _, r := range rs {
|
||||
class = catalogue.StricterClass(class, r.Class)
|
||||
names = append(names, r.Module+"/"+r.Item)
|
||||
}
|
||||
state := "could not be read"
|
||||
if red.Healthy != nil {
|
||||
state = "is NOT healthy"
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
|
||||
ID: machine + ".array." + strings.NewReplacer("/", "-", ":", "-").Replace(red.Kind+"-"+red.Where),
|
||||
Token: kindArrayDegraded, Kind: kindArrayDegraded, Machine: machine, Severity: severityOf(class),
|
||||
Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("the %s storage %s on %s %s: %s — and it is what protects %s", red.Kind, red.Where, machine,
|
||||
state, firstLine(red.Said), strings.Join(names, ", "))})
|
||||
}
|
||||
// The same item on several machines: a copy that is in use and far smaller than one kept elsewhere is
|
||||
// an empty replacement. Only against a retired copy — a module running on two machines on purpose
|
||||
// keeps two different sets of data.
|
||||
for _, key := range keysSorted(byItem) {
|
||||
rs := byItem[key]
|
||||
for _, a := range rs {
|
||||
if a.Retired() || a.Size == nil || !catalogue.Watched(a.Class) {
|
||||
continue
|
||||
}
|
||||
for _, o := range rs {
|
||||
if o.Machine == a.Machine || !o.Retired() || o.Size == nil || !replacedByLess(*a.Size, *o.Size) {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
|
||||
ID: keyOfItem(a.Machine, a.Module, a.Item), Token: kindEmptyReplacement, Kind: kindEmptyReplacement,
|
||||
Machine: a.Machine, Also: []string{o.Machine}, Severity: severityOf(a.Class), Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s of %s on %s holds %s, and the copy %s kept on %s holds %s: %s is running on "+
|
||||
"an empty replacement of its data. Move the data, or assign it back where its data is",
|
||||
a.Item, a.Module, a.Machine, sizeWords(a.Size), o.Module, o.Machine, sizeWords(o.Size), a.Module)})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, consumerFindings(copies)...)
|
||||
return out
|
||||
}
|
||||
|
||||
// replacedByLess is whether a copy in use is an empty replacement of a copy kept elsewhere: less than
|
||||
// half of it, and at least shrinkFloor less.
|
||||
func replacedByLess(inUse, kept int64) bool {
|
||||
return inUse*2 < kept && kept-inUse >= shrinkFloor
|
||||
}
|
||||
|
||||
// consumerFindings is the same question of consumers' data at providers: one consumer, the same
|
||||
// provider module on two machines.
|
||||
func consumerFindings(copies []consumerCopy) []conditions.Observation {
|
||||
by := map[string][]consumerCopy{}
|
||||
for _, c := range copies {
|
||||
k := c.Module + "/" + c.Consumer
|
||||
by[k] = append(by[k], c)
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for _, k := range keysSorted(by) {
|
||||
cs := by[k]
|
||||
if len(cs) < 2 {
|
||||
continue
|
||||
}
|
||||
found := false
|
||||
for _, a := range cs {
|
||||
if a.Retired || a.Size == nil {
|
||||
continue
|
||||
}
|
||||
for _, o := range cs {
|
||||
if o.Node == a.Node || o.Size == nil || !replacedByLess(*a.Size, *o.Size) {
|
||||
continue
|
||||
}
|
||||
state := "active"
|
||||
if o.Retired {
|
||||
state = "retired"
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider,
|
||||
ID: a.Module + "." + a.Node + "." + a.Consumer, Token: kindEmptyReplacement, Kind: kindEmptyReplacement,
|
||||
Machine: a.Node, Also: []string{o.Node}, Severity: severityOf(catalogue.StricterClass(a.Class, o.Class)),
|
||||
Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s's data at %s on %s holds %s, and its %s copy at %s on %s holds %s: the "+
|
||||
"consumer is using an empty replacement of its data (issue 273's shape). Pin it back to %s, or move "+
|
||||
"the data first", a.Consumer, a.Module, a.Node, sizeWords(a.Size), state, o.Module, o.Node,
|
||||
sizeWords(o.Size), o.Node)})
|
||||
found = true
|
||||
break
|
||||
}
|
||||
if found {
|
||||
break
|
||||
}
|
||||
}
|
||||
if found {
|
||||
continue
|
||||
}
|
||||
var active []consumerCopy
|
||||
for _, c := range cs {
|
||||
if !c.Retired {
|
||||
active = append(active, c)
|
||||
}
|
||||
}
|
||||
if len(active) >= 2 {
|
||||
var where []string
|
||||
var also []string
|
||||
for _, c := range active {
|
||||
where = append(where, c.Node+" ("+sizeWords(c.Size)+")")
|
||||
also = append(also, c.Node)
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeProvider,
|
||||
ID: active[0].Module + "." + active[0].Consumer, Token: kindDataHeldTwice, Kind: kindDataHeldTwice,
|
||||
Machine: active[0].Node, Also: also[1:], Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("%s has active data at %s on %d machines — %s — and only one is the one it uses",
|
||||
active[0].Consumer, active[0].Module, len(active), strings.Join(where, ", "))})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func keysSorted[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// ---- the `data` verb ---------------------------------------------------------------------------
|
||||
|
||||
const dataUsage = "data [--json] [--machine <name>] [--retired]"
|
||||
|
||||
// dataRow is one item as `data` lists it.
|
||||
type dataRow struct {
|
||||
Machine string `json:"machine"`
|
||||
Module string `json:"module"`
|
||||
Item string `json:"item"`
|
||||
Class string `json:"class"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Protection string `json:"protection,omitempty"`
|
||||
// Array is the redundant storage it is on and its state, where its holder could tell.
|
||||
Array string `json:"array,omitempty"`
|
||||
Unmeasured string `json:"unmeasured,omitempty"`
|
||||
SizeBytes *int64 `json:"size-bytes,omitempty"`
|
||||
LastWrite string `json:"last-write,omitempty"`
|
||||
MeasuredAt string `json:"measured-at,omitempty"`
|
||||
LastBackup string `json:"last-backup,omitempty"`
|
||||
BackupDue string `json:"backup-within,omitempty"`
|
||||
Retired string `json:"retired,omitempty"`
|
||||
RetiredWhy string `json:"retired-why,omitempty"`
|
||||
Deleted string `json:"deleted,omitempty"`
|
||||
}
|
||||
|
||||
// dataCommand is `data`: every item every machine declares, or held retired, as the self-check last
|
||||
// found it.
|
||||
func dataCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("data", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
only := set.String("machine", "", "one machine")
|
||||
retiredOnly := set.Bool("retired", false, "only what is retired")
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New(dataUsage)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
records, err := open.inventory.Data(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows := dataRows(records, shelf, *only, *retiredOnly)
|
||||
if *asJSON {
|
||||
return printJSON(map[string]any{"data": rows})
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
fmt.Println("no data on record: the self-check (D13) records what each machine declares on its next run")
|
||||
return nil
|
||||
}
|
||||
for _, r := range rows {
|
||||
state := ""
|
||||
switch {
|
||||
case r.Deleted != "":
|
||||
state = " DELETED " + r.Deleted
|
||||
case r.Retired != "":
|
||||
state = " RETIRED " + r.Retired + " — " + r.RetiredWhy
|
||||
}
|
||||
fmt.Printf("%s %s/%s %s %s %s protected by %s%s\n", r.Machine, r.Module, r.Item, r.Class,
|
||||
sizeWords(r.SizeBytes), orUnknownPath(r.Path), orNothingWord(r.Protection), state)
|
||||
if r.Array != "" {
|
||||
fmt.Printf(" on %s\n", r.Array)
|
||||
}
|
||||
if r.Unmeasured != "" {
|
||||
fmt.Printf(" not measured: %s\n", r.Unmeasured)
|
||||
}
|
||||
if r.Class == catalogue.ClassCache {
|
||||
continue
|
||||
}
|
||||
fmt.Printf(" last write %s, measured %s, last backup %s%s\n", orNever(r.LastWrite), orNever(r.MeasuredAt),
|
||||
orNever(r.LastBackup), within(r.BackupDue))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func dataRows(records []inventory.DataRecord, shelf map[string]catalogue.Manifest, only string, retiredOnly bool) []dataRow {
|
||||
rows := []dataRow{}
|
||||
stamp := func(t *time.Time) string {
|
||||
if t == nil {
|
||||
return ""
|
||||
}
|
||||
return t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
for _, r := range records {
|
||||
if only != "" && r.Machine != only {
|
||||
continue
|
||||
}
|
||||
if retiredOnly && !r.Retired() {
|
||||
continue
|
||||
}
|
||||
row := dataRow{Machine: r.Machine, Module: r.Module, Item: r.Item, Class: r.Class, Path: r.Path,
|
||||
Protection: r.Protection, Unmeasured: r.MeasureError, SizeBytes: r.Size, LastWrite: stamp(r.LastWrite), MeasuredAt: stamp(r.MeasuredAt),
|
||||
LastBackup: stamp(r.LastBackup), Retired: stamp(r.RetiredAt), RetiredWhy: r.RetiredWhy,
|
||||
Deleted: stamp(r.DeletedAt)}
|
||||
if it, ok := shelf[r.Module].DataItem(r.Item); ok && it.BackedUp() {
|
||||
row.BackupDue = it.BackupWithin().String()
|
||||
}
|
||||
if red := r.Redundancy; red != nil {
|
||||
state := "state unread"
|
||||
if red.Healthy != nil && *red.Healthy {
|
||||
state = "healthy"
|
||||
} else if red.Healthy != nil {
|
||||
state = "NOT HEALTHY"
|
||||
}
|
||||
row.Array = red.Kind + " " + red.Where + ", " + state
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func orNothingWord(s string) string {
|
||||
if s == "" || s == "none" {
|
||||
return "nothing"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func orNever(s string) string {
|
||||
if s == "" {
|
||||
return "never"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func within(s string) string {
|
||||
if s == "" {
|
||||
return " (not backed up)"
|
||||
}
|
||||
return " (bound " + s + ")"
|
||||
}
|
||||
|
||||
// ---- cleanup of retired own data ---------------------------------------------------------------
|
||||
|
||||
// The tools a node-backup holder serves to delete one retired item (novox/hq ADR 0233): the first
|
||||
// takes a last restore point of it, tagged as retired, and only then removes it — in the background,
|
||||
// because a large item outlasts any call — and the second says how that went. Module tools, not seat
|
||||
// verbs: only the controller's `cleanup delete` calls them, as it calls a provider's provisioner_delete.
|
||||
const (
|
||||
ToolDeleteRetired = "backup_delete_retired"
|
||||
ToolDeletedOutcome = "backup_deleted"
|
||||
)
|
||||
|
||||
// deletionWait is how long `cleanup delete` follows a deletion before handing it back to the person.
|
||||
var deletionWait = 8 * time.Minute
|
||||
|
||||
// deletionPoll is how often it asks.
|
||||
var deletionPoll = 5 * time.Second
|
||||
|
||||
// deletion is a holder's account of one deletion.
|
||||
type deletion struct {
|
||||
Started bool `json:"started"`
|
||||
Running bool `json:"running"`
|
||||
Done bool `json:"done"`
|
||||
OK bool `json:"ok"`
|
||||
Snapshot string `json:"snapshot"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
// retiredData is every retired item on record, as `cleanup list` shows them.
|
||||
func retiredData(records []inventory.DataRecord, now time.Time) []retiredRow {
|
||||
var out []retiredRow
|
||||
for _, r := range records {
|
||||
if !r.Retired() {
|
||||
continue
|
||||
}
|
||||
out = append(out, retiredRow{Node: r.Machine, Module: r.Module, Consumer: r.Item, Kind: retiredDataKind,
|
||||
RetiredAt: r.RetiredAt.UTC().Format(time.RFC3339), AgeDays: int(now.Sub(*r.RetiredAt).Hours() / 24),
|
||||
SizeBytes: r.Size, Why: r.RetiredWhy, Path: r.Path, Class: r.Class})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// retiredDataKind is what `cleanup list` calls a module's own retired data, beside a provider's consumer.
|
||||
const retiredDataKind = "own-data"
|
||||
|
||||
// holderOn is the module holding node-backup on a machine.
|
||||
func holderOn(ctx context.Context, inv *inventory.Inventory, machine string) (string, error) {
|
||||
held, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, h := range held {
|
||||
if s, known := catalogue.SeatNamed(h.Claim); known && s.Name == catalogue.BackupSeat && h.Node == machine {
|
||||
return h.Module, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("nothing holds %s on %s, and it is the backup holder that deletes retired data there "+
|
||||
"(after a last restore point)", catalogue.BackupSeat, machine)
|
||||
}
|
||||
|
||||
// deleteRetiredData has a machine's backup holder delete one retired item: never one declared now, and
|
||||
// never one not retired. The holder takes a last restore point of it first, so the deletion can be
|
||||
// undone until a person forgets that restore point; the record says deleted only once the holder says
|
||||
// it is.
|
||||
func deleteRetiredData(ctx context.Context, conn *nats.Conn, open *stores, r inventory.DataRecord, f handActFlags) error {
|
||||
inv := open.inventory
|
||||
if !r.Retired() {
|
||||
return fmt.Errorf("%s of %s on %s is not retired — only retired data is deleted. Nothing was done",
|
||||
r.Item, r.Module, r.Machine)
|
||||
}
|
||||
if r.Path == "" {
|
||||
return fmt.Errorf("%s of %s on %s was never measured, so where it is was never said; nothing was deleted",
|
||||
r.Item, r.Module, r.Machine)
|
||||
}
|
||||
if plan, _, err := planFor(ctx, open, r.Machine); err == nil {
|
||||
for _, m := range plan.Modules {
|
||||
if _, still := m.DataItem(r.Item); still && m.Module == r.Module {
|
||||
return fmt.Errorf("%s runs on %s again and declares %s: it is not retired any more. Nothing was done",
|
||||
r.Module, r.Machine, r.Item)
|
||||
}
|
||||
}
|
||||
}
|
||||
holder, err := holderOn(ctx, inv, r.Machine)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.record(ctx, "cleanup delete", []string{r.Machine, r.Module, r.Item})
|
||||
args := map[string]any{"module": r.Module, "item": r.Item, "path": r.Path, "confirm": r.Item,
|
||||
"why": strings.TrimSpace(*f.why), "by": link.Caller(), "via": link.ViaController}
|
||||
ask := func(tool string) (deletion, error) {
|
||||
var d deletion
|
||||
answer, err := link.AskModuleToolOn(ctx, conn, holder, tool, r.Machine, args, 25*time.Second)
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return d, fmt.Errorf("%s on %s refused: %s", holder, r.Machine, answer.Error)
|
||||
}
|
||||
return d, unmarshalAnswer(answer, &d)
|
||||
}
|
||||
d, err := ask(ToolDeleteRetired)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for waited := time.Duration(0); !d.Done && waited < deletionWait; waited += deletionPoll {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(deletionPoll):
|
||||
}
|
||||
if d, err = ask(ToolDeletedOutcome); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !d.Done:
|
||||
fmt.Printf("%s on %s is still taking the last restore point of %s and deleting it; `cleanup list` keeps "+
|
||||
"showing it until the holder says it is done — the same `cleanup delete` again reads how it went\n",
|
||||
holder, r.Machine, r.Path)
|
||||
return nil
|
||||
case !d.OK:
|
||||
return fmt.Errorf("%s on %s did NOT delete %s: %s", holder, r.Machine, r.Path, d.Error)
|
||||
}
|
||||
if err := inv.MarkDataDeleted(ctx, r.Machine, r.Module, r.Item, link.Caller(), strings.TrimSpace(*f.why), time.Now()); err != nil {
|
||||
return fmt.Errorf("%s deleted %s on %s, and it could not be recorded: %w", holder, r.Path, r.Machine, err)
|
||||
}
|
||||
fmt.Printf("%s on %s deleted %s of %s (%s, %s); its last restore point is %s, kept until a person forgets it\n",
|
||||
holder, r.Machine, r.Item, r.Module, r.Path, sizeWords(r.Size), orNever(d.Snapshot))
|
||||
return nil
|
||||
}
|
||||
|
||||
// keptOnUnassign says, for an unassignment, the irreplaceable and valuable data each module leaves in its
|
||||
// own directories on the machine:
|
||||
// kept, and retired at the self-check's next run.
|
||||
func keptOnUnassign(ctx context.Context, inv *inventory.Inventory, machine string, modules []string) []string {
|
||||
records, err := inv.Data(ctx)
|
||||
if err != nil {
|
||||
return []string{"what it leaves behind could not be read from the mesh's record: " + err.Error()}
|
||||
}
|
||||
var out []string
|
||||
for _, r := range records {
|
||||
if r.Machine != machine || r.DeletedAt != nil || !catalogue.Retires(r.Class) || !r.Owned {
|
||||
continue
|
||||
}
|
||||
for _, m := range modules {
|
||||
if r.Module == m {
|
||||
out = append(out, fmt.Sprintf("%s's %s (%s, %s) stays where it is: it is %s, so it is retired, "+
|
||||
"never removed — `cleanup list` shows it, `cleanup delete` alone removes it (novox/hq ADR 0233)",
|
||||
r.Module, r.Item, orUnknownPath(r.Path), sizeWords(r.Size), r.Class))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,430 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
func bytesOf(n int64) *int64 { return &n }
|
||||
|
||||
func when(t time.Time) *time.Time { return &t }
|
||||
|
||||
func shelfFor(t *testing.T, manifests ...string) map[string]catalogue.Manifest {
|
||||
t.Helper()
|
||||
out := map[string]catalogue.Manifest{}
|
||||
for _, raw := range manifests {
|
||||
m, err := catalogue.ParseManifest([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out[m.Module] = m
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
const houseManifest = `{"module":"house","version":"1",
|
||||
"data":{"own":[{"id":"config","path":"${dir:config}","class":"irreplaceable","active":"1d"}]},
|
||||
"resources":[{"id":"config","type":"directory","mode":"0700"}]}`
|
||||
|
||||
func findingsByKind(obs []conditions.Observation) map[string]conditions.Observation {
|
||||
out := map[string]conditions.Observation{}
|
||||
for _, o := range obs {
|
||||
out[o.Kind] = o
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// THE INCIDENT (issue 273), replayed against what D13 reads: five applications on the home server bound,
|
||||
// by one changed rule, to the store on the control node, which made each an empty database — while
|
||||
// their real databases, hundreds of megabytes each, sat on the home server's own store, by then retired
|
||||
// because the mesh no longer asked for them there. Each is an empty replacement, naming both machines
|
||||
// and the pin back: a warning for the store's consumers, whose data is valuable; urgent for one that says
|
||||
// its data there is irreplaceable (`kept-by`).
|
||||
func TestAnEmptyReplacementOfAConsumersDataIsSaid(t *testing.T) {
|
||||
var copies []consumerCopy
|
||||
for _, app := range []string{"mesh_home_board", "mesh_home_flows", "mesh_home_agents", "mesh_home_game", "mesh_home_cars"} {
|
||||
copies = append(copies,
|
||||
consumerCopy{Node: "home", Module: "postgres", Consumer: app, Size: bytesOf(400 << 20), Retired: true, Class: "valuable"},
|
||||
consumerCopy{Node: "anchor", Module: "postgres", Consumer: app, Size: bytesOf(9 << 20), Class: "valuable"})
|
||||
}
|
||||
copies[1].Class = "irreplaceable" // the photo site's own database says so
|
||||
got := dataFindings(nil, nil, nil, copies, nil, time.Now())
|
||||
if len(got) != 5 {
|
||||
t.Fatalf("%d findings for five empty replacements: %+v", len(got), got)
|
||||
}
|
||||
for i, o := range got {
|
||||
want := conditions.Warning
|
||||
if strings.Contains(o.ID, "mesh_home_board") {
|
||||
want = conditions.Urgent
|
||||
}
|
||||
_ = i
|
||||
if o.Kind != kindEmptyReplacement || o.Severity != want || o.Machine != "anchor" ||
|
||||
len(o.Also) != 1 || o.Also[0] != "home" || !strings.Contains(o.Summary, "Pin it back to home") {
|
||||
t.Errorf("%+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// While the old copy is still active (the first ten minutes), it is the same finding.
|
||||
copies[0].Retired = false
|
||||
copies[1].Class = "valuable"
|
||||
if got := dataFindings(nil, nil, nil, copies[:2], nil, time.Now()); len(got) != 1 || got[0].Kind != kindEmptyReplacement {
|
||||
t.Fatalf("with the old copy still active: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A move a person made — the data moved first, then pinned — leaves a full copy at the new provider and
|
||||
// a retired one at the old: nothing to say here; `cleanup` covers the old one.
|
||||
func TestADeliberateMoveIsNoEmptyReplacement(t *testing.T) {
|
||||
copies := []consumerCopy{
|
||||
{Node: "home", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(400 << 20), Retired: true},
|
||||
{Node: "anchor", Module: "postgres", Consumer: "mesh_home_board", Size: bytesOf(402 << 20)},
|
||||
}
|
||||
if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 {
|
||||
t.Fatalf("a deliberate move raised %+v", got)
|
||||
}
|
||||
// Two small databases differing by less than the floor are not a finding either.
|
||||
copies[0].Size, copies[1].Size = bytesOf(12<<20), bytesOf(8<<20)
|
||||
if got := dataFindings(nil, nil, nil, copies, nil, time.Now()); len(got) != 0 {
|
||||
t.Fatalf("noise between two empty databases raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Where a provider cannot say sizes, a consumer active at two providers is still said — as a warning,
|
||||
// since which one is empty cannot be told.
|
||||
func TestConsumerDataActiveTwiceWithoutSizesIsAWarning(t *testing.T) {
|
||||
copies := []consumerCopy{
|
||||
{Node: "home", Module: "minio", Consumer: "mesh_home_photos"},
|
||||
{Node: "anchor", Module: "minio", Consumer: "mesh_home_photos"},
|
||||
}
|
||||
got := dataFindings(nil, nil, nil, copies, nil, time.Now())
|
||||
if len(got) != 1 || got[0].Kind != kindDataHeldTwice || got[0].Severity != conditions.Warning {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The same incident for a module's own data: a module unassigned from one machine and assigned on
|
||||
// another starts over in an empty directory while its full one is kept, retired, where it was.
|
||||
func TestAnEmptyReplacementOfAModulesOwnDataIsUrgent(t *testing.T) {
|
||||
now := time.Now()
|
||||
retired := now.Add(-time.Hour)
|
||||
records := []inventory.DataRecord{
|
||||
{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config",
|
||||
Size: bytesOf(2 << 30), RetiredAt: &retired, FirstSeen: now.Add(-90 * 24 * time.Hour)},
|
||||
{Machine: "anchor", Module: "house", Item: "config", Class: "irreplaceable", Path: "/var/lib/house/config",
|
||||
Size: bytesOf(1 << 20), FirstSeen: now.Add(-time.Hour), LastWrite: when(now)},
|
||||
}
|
||||
got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))
|
||||
o, ok := got[kindEmptyReplacement]
|
||||
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "home" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
// The same of a valuable item is a warning.
|
||||
records[0].Class, records[1].Class = "valuable", "valuable"
|
||||
if o := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now))[kindEmptyReplacement]; o.Severity != conditions.Warning {
|
||||
t.Fatalf("a valuable empty replacement: %+v", o)
|
||||
}
|
||||
records[0].Class, records[1].Class = "irreplaceable", "irreplaceable"
|
||||
// Two machines running a module on purpose, both active, keep two sets of data: nothing to say.
|
||||
records[0].RetiredAt = nil
|
||||
if got := findingsByKind(dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now)); got[kindEmptyReplacement].Kind != "" {
|
||||
t.Fatalf("two active copies were read as a replacement: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An irreplaceable item that lost more than half of its largest size in a week is urgent; a smaller loss,
|
||||
// or a loss under the floor, is not a finding.
|
||||
func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) {
|
||||
now := time.Now()
|
||||
r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
|
||||
Size: bytesOf(300 << 20), FirstSeen: now.Add(-30 * 24 * time.Hour), LastWrite: when(now), LastBackup: when(now)}
|
||||
shelf := shelfFor(t, houseManifest)
|
||||
o := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): 1 << 30}, shelf, nil, nil, now))[kindDataShrank]
|
||||
if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "shrank") {
|
||||
t.Fatalf("%+v", o)
|
||||
}
|
||||
for _, peak := range []int64{500 << 20, 20 << 20} {
|
||||
if got := findingsByKind(dataFindings([]inventory.DataRecord{r}, map[string]int64{r.Key(): peak}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" {
|
||||
t.Errorf("a peak of %d raised a shrink", peak)
|
||||
}
|
||||
}
|
||||
small := r
|
||||
small.Size = bytesOf(1 << 20)
|
||||
if got := findingsByKind(dataFindings([]inventory.DataRecord{small}, map[string]int64{r.Key(): 10 << 20}, shelf, nil, nil, now)); got[kindDataShrank].Kind != "" {
|
||||
t.Error("a loss under the floor raised a shrink")
|
||||
}
|
||||
}
|
||||
|
||||
// Data said to be written all the time and not written; data with no backup or an old one — urgent when
|
||||
// irreplaceable, a warning when valuable; and a new item given its bound before it is said.
|
||||
func TestQuietDataAndMissingBackupsAreSaidByClass(t *testing.T) {
|
||||
now := time.Now()
|
||||
shelf := shelfFor(t, houseManifest)
|
||||
r := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
|
||||
Size: bytesOf(1 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), LastWrite: when(now.Add(-3 * 24 * time.Hour)),
|
||||
LastBackup: when(now.Add(-72 * time.Hour))}
|
||||
got := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now))
|
||||
if got[kindDataQuiet].Severity != conditions.Urgent || got[kindBackupStale].Severity != conditions.Urgent {
|
||||
t.Fatalf("irreplaceable: %+v", got)
|
||||
}
|
||||
valuable := r
|
||||
valuable.Class, valuable.MeasuredAt = "valuable", when(now) // measured: a holder is there to take its backup
|
||||
if got := findingsByKind(dataFindings([]inventory.DataRecord{valuable}, nil, shelf, nil, nil, now)); got[kindDataQuiet].Severity != conditions.Warning ||
|
||||
got[kindBackupStale].Severity != conditions.Warning {
|
||||
t.Fatalf("valuable: %+v", got)
|
||||
}
|
||||
never := r
|
||||
never.LastBackup = nil
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{never}, nil, shelf, nil, nil, now))[kindBackupStale]; !strings.Contains(o.Summary, "no good backup") {
|
||||
t.Fatalf("never backed up: %+v", o)
|
||||
}
|
||||
fresh := never
|
||||
fresh.FirstSeen, fresh.LastWrite = now.Add(-time.Hour), when(now)
|
||||
if got := dataFindings([]inventory.DataRecord{fresh}, nil, shelf, nil, nil, now); len(got) != 0 {
|
||||
t.Fatalf("an item declared an hour ago, before its first night, raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An item retired more than thirty days waits for a person; less, it is only listed.
|
||||
func TestRetiredDataWaitingThirtyDaysIsSaid(t *testing.T) {
|
||||
now := time.Now()
|
||||
old, recent := now.Add(-31*24*time.Hour), now.Add(-2*24*time.Hour)
|
||||
records := []inventory.DataRecord{
|
||||
{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &old},
|
||||
{Machine: "home", Module: "attic", Item: "boxes", Class: "irreplaceable", Size: bytesOf(1 << 30), RetiredAt: &recent},
|
||||
}
|
||||
got := dataFindings(records, nil, shelfFor(t, houseManifest), nil, nil, now)
|
||||
if len(got) != 1 || got[0].Kind != kindCleanupWaiting || !strings.Contains(got[0].Summary, "cleanup delete home house config") {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if rows := retiredData(records, now); len(rows) != 2 || rows[0].Kind != retiredDataKind {
|
||||
t.Fatalf("cleanup list: %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// The holder's answer reads into measurements, by module and item.
|
||||
func TestTheHoldersAnswerIsRead(t *testing.T) {
|
||||
raw := []byte(`[{"module":"postgres","runs":1,"paths":["/var/lib/mesh-store/dumps"],"lastNight":null,"restorePoints":3,
|
||||
"data":[{"item":"store","class":"irreplaceable","path":"/var/lib/mesh-store","covered_by":"/var/lib/mesh-store/dumps",
|
||||
"size_bytes":1073741824,"last_write":"2026-10-06T10:00:00Z","measured_at":"2026-10-06T10:05:00Z","last_backup":"2026-10-06T03:10:00Z"}]}]`)
|
||||
got, err := readHolder(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := got["postgres"]["store"]
|
||||
if m.Path != "/var/lib/mesh-store" || m.Size == nil || *m.Size != 1<<30 || m.LastBackup == nil || m.MeasuredAt == nil {
|
||||
t.Fatalf("%+v", m)
|
||||
}
|
||||
// An older holder, which says no data, reads as nothing measured rather than a failure.
|
||||
if got, err := readHolder([]byte(`[{"module":"postgres","runs":1,"paths":[]}]`)); err != nil || len(got) != 0 {
|
||||
t.Fatalf("%v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeHolder answers node-backup's `backed-up` on one machine over a real bus, with what it is told it
|
||||
// measured.
|
||||
type fakeHolder struct {
|
||||
mu sync.Mutex
|
||||
modules []map[string]any
|
||||
}
|
||||
|
||||
func (f *fakeHolder) set(modules ...map[string]any) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.modules = modules
|
||||
}
|
||||
|
||||
func (f *fakeHolder) serve(t *testing.T, conn *nats.Conn, node string) {
|
||||
t.Helper()
|
||||
sub, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
body, _ := json.Marshal(map[string]any{"result": f.modules, "error": "", "node": node})
|
||||
_ = m.Respond(body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
if err := conn.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func measuredHouse(path string, size int64, at time.Time) map[string]any {
|
||||
return map[string]any{"module": "house", "runs": 0, "paths": []string{path}, "data": []map[string]any{{
|
||||
"item": "config", "class": "irreplaceable", "path": path, "covered_by": path, "size_bytes": size,
|
||||
"last_write": at, "measured_at": at, "last_backup": at}}}
|
||||
}
|
||||
|
||||
// UNASSIGNING A MODULE WITH IRREPLACEABLE DATA KEEPS THE DATA, and assigning it elsewhere onto an empty
|
||||
// directory is an empty replacement — through the real stores and a real bus. The unassignment says the
|
||||
// data stays; the self-check's next run retires it (kept, listed by cleanup), and when the module comes
|
||||
// up on another machine with an empty directory while the full one waits retired, that is urgent.
|
||||
func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
||||
house, err := catalogue.ParseManifest([]byte(houseManifest))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, house)
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err == nil {
|
||||
t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up")
|
||||
}
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
if _, err := assign(ctx, open, node, "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn := onATestBus(t)
|
||||
js, err := broker.Dial(os.Getenv("MESH_TEST_NATS"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
laptop, anchor := &fakeHolder{}, &fakeHolder{}
|
||||
laptop.serve(t, conn, "laptop")
|
||||
anchor.serve(t, conn, "anchor")
|
||||
now := time.Now()
|
||||
heard := &watchdogs{last: &signalFacts{now: now, machines: []machineFacts{
|
||||
{name: "laptop", lastHeard: now}, {name: "anchor", lastHeard: now}}}}
|
||||
d := &doctor{open: open, js: js, watchdogs: heard}
|
||||
var d13 probe
|
||||
for _, p := range probeRegistry {
|
||||
if p.ID == probeDataID {
|
||||
d13 = p
|
||||
}
|
||||
}
|
||||
run := func() []conditions.Observation {
|
||||
t.Helper()
|
||||
probing := context.WithValue(ctx, probeAsksKey{}, d13)
|
||||
obs, err := probeData(probing, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return obs
|
||||
}
|
||||
|
||||
laptop.set(measuredHouse("/var/lib/house/config", 2<<30, now))
|
||||
if obs := run(); len(obs) != 0 {
|
||||
t.Fatalf("a measured, backed-up item raised %+v", obs)
|
||||
}
|
||||
r, err := inv.DataOf(ctx, "laptop", "house", "config")
|
||||
if err != nil || r.Path != "/var/lib/house/config" || r.Size == nil || *r.Size != 2<<30 || r.LastBackup == nil {
|
||||
t.Fatalf("what the holder measured was not kept: %+v, %v", r, err)
|
||||
}
|
||||
|
||||
said, err := unassign(ctx, open, "laptop", "house")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "house's config (/var/lib/house/config, 2.0 GB) stays where it is") {
|
||||
t.Fatalf("the unassignment does not say the data stays:\n%s", said)
|
||||
}
|
||||
laptop.set()
|
||||
run()
|
||||
r, err = inv.DataOf(ctx, "laptop", "house", "config")
|
||||
if err != nil || !r.Retired() || r.Path != "/var/lib/house/config" {
|
||||
t.Fatalf("unassigned, the irreplaceable item is not kept retired: %+v, %v", r, err)
|
||||
}
|
||||
records, _ := inv.Data(ctx)
|
||||
if rows := retiredData(records, time.Now()); len(rows) != 1 || rows[0].Path != "/var/lib/house/config" {
|
||||
t.Fatalf("cleanup list: %+v", rows)
|
||||
}
|
||||
|
||||
// Assigned on the anchor, onto an empty directory.
|
||||
if _, err := assign(ctx, open, "anchor", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
anchor.set(measuredHouse("/var/lib/house/config", 300<<10, time.Now()))
|
||||
obs := findingsByKind(run())
|
||||
o, ok := obs[kindEmptyReplacement]
|
||||
if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "laptop" {
|
||||
t.Fatalf("an empty replacement of a module's data was not urgent: %+v", obs)
|
||||
}
|
||||
}
|
||||
|
||||
// Data on redundant storage: the array it is on is watched, one condition per array as loud as the most
|
||||
// precious item on it; an item said to be on redundancy and found on plain storage is said; an item
|
||||
// whose path is gone is said.
|
||||
func TestTheArrayUnderDataIsWatched(t *testing.T) {
|
||||
now := time.Now()
|
||||
media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}],
|
||||
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array, no room to copy"},
|
||||
{"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array, no room to copy"}]}}`
|
||||
shelf := shelfFor(t, media)
|
||||
sick := false
|
||||
on := func(item string, healthy *bool) inventory.DataRecord {
|
||||
return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable", Owned: false,
|
||||
Path: "/tank/" + item, Size: bytesOf(40 << 40), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now),
|
||||
Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: healthy, Said: "pool 'tank' is DEGRADED"}}
|
||||
}
|
||||
got := dataFindings([]inventory.DataRecord{on("films", &sick), on("shows", &sick)}, nil, shelf, nil, nil, now)
|
||||
if len(got) != 1 || got[0].Kind != kindArrayDegraded || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "media/films, media/shows") {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
well := true
|
||||
if got := dataFindings([]inventory.DataRecord{on("films", &well)}, nil, shelf, nil, nil, now); len(got) != 0 {
|
||||
t.Fatalf("a healthy array raised %+v", got)
|
||||
}
|
||||
plain := on("films", nil)
|
||||
plain.Redundancy = nil
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{plain}, nil, shelf, nil, nil, now))[kindProtectionMissing]; o.Severity != conditions.Urgent {
|
||||
t.Fatalf("redundancy said and not found: %+v", o)
|
||||
}
|
||||
gone := on("films", &well)
|
||||
gone.MeasureError, gone.Size = "/tank/films does not exist", bytesOf(0)
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{gone}, map[string]int64{gone.Key(): 40 << 40}, shelf, nil, nil, now))[kindDataMissing]; o.Severity != conditions.Urgent {
|
||||
t.Fatalf("a vanished library: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// What a consumer keeps with its provider as irreplaceable holds the provider's item to that class:
|
||||
// the photo site's objects make the object store's data an urgent matter.
|
||||
func TestKeptByHoldsTheProvidersItemToTheConsumersClass(t *testing.T) {
|
||||
objects := `{"module":"objects","version":"1","provides":[{"name":"s3-bucket","scope":"mesh"}],"grants":{"s3-bucket":"${dir:g}"},
|
||||
"data":{"own":[{"id":"data","path":"${dir:data}","class":"valuable"}],"consumers":{"s3-bucket":{"class":"valuable","in":"data"}}},
|
||||
"resources":[{"id":"g","type":"directory","mode":"0700"},{"id":"data","type":"directory","mode":"0700"}]}`
|
||||
photos := `{"module":"photos","version":"1","requires":["s3-bucket"],"data":{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}}`
|
||||
shelf := shelfFor(t, objects, photos)
|
||||
bindings := []inventory.Binding{{Machine: "anchor", Consumer: "photos", Provision: "s3-bucket",
|
||||
Provider: catalogue.Chosen{Node: "anchor", Module: "objects"}}}
|
||||
upgraded, keptBy := keptByClasses(bindings, shelf)
|
||||
if upgraded["anchor/objects/data"] != "irreplaceable" || keptBy["objects/mesh_anchor_photos"] != "irreplaceable" {
|
||||
t.Fatalf("upgraded %v, kept by %v", upgraded, keptBy)
|
||||
}
|
||||
now := time.Now()
|
||||
r := inventory.DataRecord{Machine: "anchor", Module: "objects", Item: "data", Class: "valuable", Owned: true,
|
||||
Size: bytesOf(10 << 30), FirstSeen: now.Add(-10 * 24 * time.Hour), MeasuredAt: when(now), LastBackup: when(now.Add(-72 * time.Hour))}
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, upgraded, now))[kindBackupStale]; o.Severity != conditions.Urgent {
|
||||
t.Fatalf("the photos' store without a backup: %+v", o)
|
||||
}
|
||||
if o := findingsByKind(dataFindings([]inventory.DataRecord{r}, nil, shelf, nil, nil, now))[kindBackupStale]; o.Severity != conditions.Warning {
|
||||
t.Fatalf("a valuable store without a backup: %+v", o)
|
||||
}
|
||||
}
|
||||
@@ -102,6 +102,14 @@ var probeRegistry = []probe{
|
||||
{ID: probeBindingsID, Asserts: "every consumer of a provision that keeps its data is bound where it was last " +
|
||||
"sent, or moves by a pin", From: "issue 273, ADR 0232", Kind: kindBindingMoved,
|
||||
Raises: []string{kindBindingKept, kindBindingMoving}, Phase: 2, run: probeBindings},
|
||||
{ID: probeDataID, Asserts: "every item of data a machine declares is measured, is there, holds what it held, is " +
|
||||
"written where it should be, is backed up within its bound or sits on healthy redundant storage, and is no " +
|
||||
"empty replacement of a copy kept elsewhere; what a machine no longer declares that is irreplaceable or " +
|
||||
"valuable is retired, not forgotten", From: "issue 273, ADR 0233",
|
||||
Kind: kindDataShrank, Raises: []string{kindEmptyReplacement, kindDataHeldTwice, kindDataQuiet, kindBackupStale,
|
||||
kindDataUnmeasured, kindDataMissing, kindArrayDegraded, kindProtectionMissing, kindCleanupWaiting},
|
||||
Phase: 2, run: probeData,
|
||||
Asks: []broker.SeatVerb{{Seat: "node-backup", Verb: "backed-up"}}},
|
||||
{ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals",
|
||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs},
|
||||
}
|
||||
|
||||
@@ -170,6 +170,9 @@ func run() error {
|
||||
return retireCommand(ctx, args[1:])
|
||||
case "cleanup":
|
||||
return cleanupCommand(ctx, args[1:])
|
||||
// The data every machine declares, as the self-check last found it (novox/hq ADR 0233).
|
||||
case "data":
|
||||
return dataCommand(ctx, args[1:])
|
||||
case "version":
|
||||
fmt.Println(version)
|
||||
return nil
|
||||
|
||||
@@ -258,10 +258,21 @@ func cleanupCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
defer open.Close()
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
return deleteOlderThan(ctx, open.inventory, conn, *olderThan, *confirm, f, time.Now())
|
||||
return deleteOlderThan(ctx, open, conn, *olderThan, *confirm, f, time.Now())
|
||||
})
|
||||
case *olderThan == 0 && len(rest) == 3 && !*confirm:
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
// A module's own retired data, when the mesh holds such an item (novox/hq ADR 0233); a
|
||||
// provider's retired consumer otherwise.
|
||||
if r, err := open.inventory.DataOf(ctx, rest[0], rest[1], rest[2]); err == nil && r.DeletedAt == nil &&
|
||||
r.RetiredAt != nil {
|
||||
return deleteRetiredData(ctx, conn, open, r, f)
|
||||
}
|
||||
return deleteRetired(ctx, conn, providerInstance{Node: rest[0], Module: rest[1]}, rest[2], f)
|
||||
})
|
||||
}
|
||||
@@ -285,6 +296,10 @@ type retiredRow struct {
|
||||
Why string `json:"why,omitempty"`
|
||||
// Access is "kept" for a consumer of a mark-only provider: retired on record, still reachable.
|
||||
Access string `json:"access,omitempty"`
|
||||
// Path and Class are a module's own retired data's (Kind own-data, novox/hq ADR 0233): where it is
|
||||
// kept on its machine, and its class. Consumer is then the item.
|
||||
Path string `json:"path,omitempty"`
|
||||
Class string `json:"class,omitempty"`
|
||||
}
|
||||
|
||||
// retiredListing is every provider's retired consumers, and the providers that could not say.
|
||||
@@ -299,7 +314,15 @@ func gatherRetired(ctx context.Context, inv *inventory.Inventory, conn *nats.Con
|
||||
if err != nil {
|
||||
return retiredListing{}, err
|
||||
}
|
||||
return retiredOf(ctx, conn, instances, now), nil
|
||||
listing := retiredOf(ctx, conn, instances, now)
|
||||
// And every module's own data retired on its machine (novox/hq ADR 0233).
|
||||
records, err := inv.Data(ctx)
|
||||
if err != nil {
|
||||
return retiredListing{}, err
|
||||
}
|
||||
listing.Retired = append(listing.Retired, retiredData(records, now)...)
|
||||
sort.SliceStable(listing.Retired, func(i, j int) bool { return listing.Retired[i].AgeDays > listing.Retired[j].AgeDays })
|
||||
return listing, nil
|
||||
}
|
||||
|
||||
func retiredOf(ctx context.Context, conn *nats.Conn, instances []providerInstance, now time.Time) retiredListing {
|
||||
@@ -332,7 +355,7 @@ func printRetired(l retiredListing, asJSON bool) error {
|
||||
return printJSON(l)
|
||||
}
|
||||
if len(l.Retired) == 0 {
|
||||
fmt.Println("no provider holds a retired consumer")
|
||||
fmt.Println("no provider holds a retired consumer, and no machine holds retired data")
|
||||
}
|
||||
for _, r := range l.Retired {
|
||||
age := "age unknown"
|
||||
@@ -346,6 +369,11 @@ func printRetired(l retiredListing, asJSON bool) error {
|
||||
if r.Access == "kept" {
|
||||
kind += " [MARK ONLY: access kept until deleted]"
|
||||
}
|
||||
if r.Kind == retiredDataKind {
|
||||
fmt.Printf("%s on %s: its own %s, %s, at %s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer,
|
||||
r.Class, r.Path, age, sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why))
|
||||
continue
|
||||
}
|
||||
fmt.Printf("%s on %s: %s%s — retired %s, %s, %s\n %s\n", r.Module, r.Node, r.Consumer, kind, age,
|
||||
sizeWords(r.SizeBytes), r.RetiredAt, orWhy("", r.Why))
|
||||
}
|
||||
@@ -391,16 +419,16 @@ func deleteRetired(ctx context.Context, conn *nats.Conn, p providerInstance, con
|
||||
|
||||
// deleteOlderThan lists every consumer retired more than days ago, and deletes them only with confirm.
|
||||
// One whose age the provider cannot say is never in it.
|
||||
func deleteOlderThan(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, days int, confirm bool,
|
||||
func deleteOlderThan(ctx context.Context, open *stores, conn *nats.Conn, days int, confirm bool,
|
||||
f handActFlags, now time.Time) error {
|
||||
listing, err := gatherRetired(ctx, inv, conn, now)
|
||||
listing, err := gatherRetired(ctx, open.inventory, conn, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return deleteFrom(ctx, conn, listing, days, confirm, f)
|
||||
return deleteFrom(ctx, conn, open, listing, days, confirm, f)
|
||||
}
|
||||
|
||||
func deleteFrom(ctx context.Context, conn *nats.Conn, listing retiredListing, days int, confirm bool, f handActFlags) error {
|
||||
func deleteFrom(ctx context.Context, conn *nats.Conn, open *stores, listing retiredListing, days int, confirm bool, f handActFlags) error {
|
||||
var due []retiredRow
|
||||
unknown := 0
|
||||
for _, r := range listing.Retired {
|
||||
@@ -431,7 +459,16 @@ func deleteFrom(ctx context.Context, conn *nats.Conn, listing retiredListing, da
|
||||
}
|
||||
var failed []string
|
||||
for _, r := range due {
|
||||
if err := deleteRetired(ctx, conn, providerInstance{Node: r.Node, Module: r.Module}, r.Consumer, f); err != nil {
|
||||
var err error
|
||||
if r.Kind == retiredDataKind {
|
||||
var rec inventory.DataRecord
|
||||
if rec, err = open.inventory.DataOf(ctx, r.Node, r.Module, r.Consumer); err == nil {
|
||||
err = deleteRetiredData(ctx, conn, open, rec, f)
|
||||
}
|
||||
} else {
|
||||
err = deleteRetired(ctx, conn, providerInstance{Node: r.Node, Module: r.Module}, r.Consumer, f)
|
||||
}
|
||||
if err != nil {
|
||||
failed = append(failed, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -380,12 +380,12 @@ func TestNatsCleanupDeletesOnlyTheNamedRetiredConsumer(t *testing.T) {
|
||||
t.Fatalf("%+v", listing)
|
||||
}
|
||||
fake.deleted = nil
|
||||
said = printed(t, func() error { return deleteFrom(t.Context(), conn, listing, 30, false, whyFlags(t, "tidy")) })
|
||||
said = printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, false, whyFlags(t, "tidy")) })
|
||||
if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") ||
|
||||
strings.Contains(said, "young") {
|
||||
t.Fatalf("deleted %v; said %s", fake.deleted, said)
|
||||
}
|
||||
printed(t, func() error { return deleteFrom(t.Context(), conn, listing, 30, true, whyFlags(t, "tidy")) })
|
||||
printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, true, whyFlags(t, "tidy")) })
|
||||
if !slices.Equal(fake.deleted, []string{"old"}) {
|
||||
t.Fatalf("confirmed, deleted %v", fake.deleted)
|
||||
}
|
||||
|
||||
@@ -499,6 +499,15 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
return argv, nil
|
||||
}
|
||||
return []string{"cleanup", "list", "--json"}, nil
|
||||
case "data":
|
||||
argv := []string{"data", "--json"}
|
||||
if m := str("machine"); m != "" {
|
||||
argv = append(argv, "--machine", m)
|
||||
}
|
||||
if on("retired") {
|
||||
argv = append(argv, "--retired")
|
||||
}
|
||||
return argv, nil
|
||||
case "doctor":
|
||||
which := 0
|
||||
argv := []string{"doctor"}
|
||||
@@ -598,7 +607,7 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
|
||||
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
|
||||
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true,
|
||||
"hand-acts": true, "durations": true, "conditions": true, "doctor": true, "retire": true, "cleanup": true}
|
||||
"hand-acts": true, "durations": true, "conditions": true, "doctor": true, "retire": true, "cleanup": true, "data": true}
|
||||
|
||||
// repairingCommand names a command line that repairs by hand, and so says why: a push, a plan stopped
|
||||
// or closed, a consumer re-made (novox/hq to-be 45 §7). Empty for any other.
|
||||
|
||||
@@ -276,6 +276,7 @@ var accountedFlags = map[string]map[string]string{
|
||||
"conditions": {"json": "set by the verb: the answer is data"},
|
||||
"retire": {"json": "set by the verb: the answer is data"},
|
||||
"cleanup": {"json": "set by the verb: the answer is data"},
|
||||
"data": {"json": "set by the verb: the answer is data"},
|
||||
"conditions history": {"json": "set by the verb: the answer is data"},
|
||||
"conditions show": {"json": "set by the verb: the answer is data"},
|
||||
"healers": {"json": "set by the verb: the answer is data"},
|
||||
|
||||
Reference in New Issue
Block a user