Design 47's last milestones: the keeper, self-check, healers, hand-acts, provisioner, vault, artifact sweep, disk readings, credential forms (issues 774, 1952, 1953, 1954, 1955)

One commit with exactly the tree of the reviewed head 35b4509c0a, on main 12c2574c9d, so two compiled programs added and removed on the branch leave no trace in history (issue 1577).

modules/restic/cmd/restic-backups/.gitignore

@@ -0,0 +1 @@
+restic-backups

modules/restic/cmd/restic-backups/backups.go

@@ -25,6 +25,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"git.novox.be/novox/mesh-sdk/go/health"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -32,6 +33,7 @@ import (
 	"slices"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 )
@@ -188,6 +190,15 @@ type Backups struct {
 	// measuring run as processes of their own). Node is the machine's name, said in the events.
 	Emit func(event string, body any) error
 	Node string
+	// Readings is every reading disk-load keeps for this machine (its state `readings`, ADR 0571 clause 3); nil where the
+	// process has no runtime to read it.
+	Readings func() ([]health.Reading, error)
+	// PutGood writes one row of the state `good` (health.GoodRow): the night's process through its own bus account, the
+	// tools through the runtime; nil where there is no bus to write to.
+	PutGood func(key string, row health.GoodRow) error
+
+	goodMu    sync.Mutex
+	goodWrote map[string][]byte
 }
 
 // The locks, files in this module's state. The night runs in a process of its own and `now`, `restore` and a

modules/restic/cmd/restic-backups/data.go

@@ -1,26 +1,9 @@
-// The data the modules on this machine declare, measured (novox/hq ADR 0233).
+// The data the modules on this machine declare (novox/hq ADR 0233), and what the readings say of them.
 //
-// The mesh composes a second file beside the backup lines: every data item every module on the
-// machine declares, one line each,
-//
-//	item <id> <class> <path> <covered-by> <protection> <measure>
-//
-// where covered-by is the path whose snapshot keeps it (the item itself, or the directory its dump
-// writes into), or `-` when it is not backed up; protection is backup, redundancy, both, or none; and
-// measure is how the item is measured. This holder measures each item that is not a cache — its size,
-// its newest write, and the redundant storage it is on and whether that is healthy (ZFS, md, btrfs) —
-// and says it, with each module's last good backup and the precision of the measurement, in
-// `backed-up`.
-//
-// **Never an unbounded walk of something large.** Three methods, the item's own choice:
-//
-//   - `walk` (the default, for small items): every file summed and the newest change found, in one walk
-//     as root — at most once a day, and stopped after walkFor or walkFiles, said as "measured partially";
-//   - `dataset`: the ZFS dataset holding the path — its `used` from the filesystem's own counters — and
-//     the newest change among the path's top-level entries; hourly, and nothing is walked;
-//   - `shallow`: the newest change among the top-level entries only, and no size; hourly. The controller keeps the readings and says when an item shrinks, stops being written,
-//
-// goes without a backup, or is replaced by an empty copy of itself; this holder only measures.
+// The mesh composes a file beside the backup lines: every data item every module on the machine declares, one line
+// each, which mesh-sdk reads (catalogue.ParseDataItems). disk-load measures the items and keeps each reading and its
+// seven-day peak as its own state (ADR 0571 clause 3); this holder reads them there and says each, with each module's
+// last good backup, in `backed-up`.
 //
 // And it deletes, when a person has decided: an item the mesh retired — its module gone from this
 // machine — is removed by `backup_delete_retired`, and only after a last restore point of it has been
@@ -32,104 +15,32 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	"git.novox.be/novox/mesh-sdk/go/health"
 	"os"
 	"path/filepath"
 	"regexp"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
 )
 
-// Item is one data item a module declares.
-type Item struct {
-	// Measure is walk, dataset or shallow.
-	Measure    string `json:"measure,omitempty"`
-	Module     string `json:"-"`
-	ID         string `json:"item"`
-	Class      string `json:"class"`
-	Path       string `json:"path"`
-	CoveredBy  string `json:"covered_by,omitempty"`
-	Protection string `json:"protection,omitempty"`
-}
+// Item is one data item a module declares, as placed on this machine (mesh-sdk's reading of the holder's data text).
+type Item = catalogue.PlacedDataItem
 
-// Redundancy is the redundant storage an item is on, as read here.
-type Redundancy struct {
-	// Kind is zfs, md or btrfs.
-	Kind string `json:"kind"`
-	// Where is the pool, the array device or the filesystem.
-	Where string `json:"where"`
-	// Healthy is nil when its state could not be read.
-	Healthy *bool  `json:"healthy"`
-	Said    string `json:"said"`
-}
-
-// Measured is what one measurement found.
-type Measured struct {
-	SizeBytes  *int64     `json:"size_bytes,omitempty"`
-	LastWrite  *time.Time `json:"last_write,omitempty"`
-	MeasuredAt *time.Time `json:"measured_at,omitempty"`
-	// Precision says what the size is: "exact", "dataset <name>: its whole size", "partial: …" (a lower
-	// bound, never compared), or "no size: …".
-	Precision  string      `json:"precision,omitempty"`
-	Error      string      `json:"error,omitempty"`
-	Redundancy *Redundancy `json:"redundancy,omitempty"`
-}
-
-// ItemReport is one item as `backed-up` says it.
+// ItemReport is one item as `backed-up` says it: the item, its newest reading from disk-load (ADR 0571 clause 3) and
+// its newest good backup.
 type ItemReport struct {
 	Item
-	Measured
+	health.Measured
+	// Peak is the largest size in the last seven days, on PeakDay.
+	Peak       *int64     `json:"peak_bytes,omitempty"`
+	PeakDay    string     `json:"peak_day,omitempty"`
 	LastBackup *time.Time `json:"last_backup,omitempty"`
 }
 
-// The classes the mesh declares; a cache is listed and never measured.
-const classCache = "cache"
-
 var safePath = regexp.MustCompile(`^/[^\s'"\\$` + "`" + `]*$`)
 
-// parseItems reads the composed data file into each module's items. A line this holder does not read
-// is refused, naming the module, as a backup line is.
-func parseItems(text string) (map[string][]Item, error) {
-	out := map[string][]Item{}
-	module := ""
-	for _, raw := range strings.Split(text, "\n") {
-		line := strings.TrimSpace(raw)
-		if line == "" {
-			continue
-		}
-		if m := moduleHeader.FindStringSubmatch(line); m != nil {
-			module = m[1]
-			continue
-		}
-		if strings.HasPrefix(line, "#") || module == "" {
-			continue
-		}
-		f := strings.Fields(line)
-		if len(f) < 5 || len(f) > 7 || f[0] != "item" || !safePath.MatchString(f[3]) || (f[4] != "-" && !safePath.MatchString(f[4])) {
-			return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line)
-		}
-		it := Item{Module: module, ID: f[1], Class: f[2], Path: f[3]}
-		if len(f) >= 6 {
-			it.Protection = f[5]
-		}
-		it.Measure = measureWalk
-		if len(f) == 7 {
-			it.Measure = f[6]
-		}
-		switch it.Measure {
-		case measureWalk, measureDataset, measureShallow:
-		default:
-			return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line)
-		}
-		if f[4] != "-" {
-			it.CoveredBy = f[4]
-		}
-		out[module] = append(out[module], it)
-	}
-	return out, nil
-}
-
 // Items is what the modules on this machine declare; none when the mesh composed no data file — an
 // older controller, which composes none.
 func (b *Backups) Items() (map[string][]Item, error) {
@@ -143,257 +54,30 @@ func (b *Backups) Items() (map[string][]Item, error) {
 	if err != nil {
 		return nil, err
 	}
-	return parseItems(string(raw))
+	return catalogue.ParseDataItems(string(raw))
 }
 
-func (b *Backups) measuredFile() string { return filepath.Join(b.Where.State, "measured.json") }
-
-// Measurements is the newest measurement of each item, by module and item.
-func (b *Backups) Measurements() map[string]map[string]Measured {
-	out := map[string]map[string]Measured{}
-	if raw, err := os.ReadFile(b.measuredFile()); err == nil {
-		_ = json.Unmarshal(raw, &out)
+// Measurements is the newest reading disk-load keeps of each item on this machine, by module and item (its state
+// `readings`, ADR 0571 clause 3); none where it cannot be read, which `backed-up` then says as no reading.
+func (b *Backups) Measurements() map[string]map[string]health.Reading {
+	out := map[string]map[string]health.Reading{}
+	if b.Readings == nil {
+		return out
+	}
+	rs, err := b.Readings()
+	if err != nil {
+		b.Say("disk-load's readings could not be read: %v", err)
+		return out
+	}
+	for _, r := range rs {
+		if out[r.Module] == nil {
+			out[r.Module] = map[string]health.Reading{}
+		}
+		out[r.Module][r.ID] = r
 	}
 	return out
 }
 
-// The ways an item is measured.
-const (
-	measureWalk    = "walk"
-	measureDataset = "dataset"
-	measureShallow = "shallow"
-)
-
-// The bounds on a walk, and how often one runs: a walk is for small items, and one that meets a large
-// one stops and says so rather than loading the disks for hours.
-var (
-	walkFor   = 10 * time.Minute
-	walkFiles = 2_000_000
-	walkEvery = 23 * time.Hour
-)
-
-// walkCommand sums the files under a path and finds its newest change, in one walk, as root — bounded
-// by time and by files. One line out: "<bytes> <epoch seconds> <files> <complete 1|0>".
-func walkCommand(path string) string {
-	return fmt.Sprintf("timeout %d find '%s' -xdev -printf '%%y %%s %%T@\\n' 2>/dev/null | head -n %d | "+
-		"awk 'BEGIN{s=0;m=0;n=0} {n++; if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%%d %%.0f %%d\\n\", s, m, n}'; "+
-		"echo \"${PIPESTATUS[0]:-0}\"", int(walkFor.Seconds()), path, walkFiles)
-}
-
-// topCommand is the newest change among a path and its top-level entries, and how many there are:
-// one directory read, bounded. "<epoch seconds> <entries>".
-func topCommand(path string) string {
-	return "timeout 60 find '" + path + "' -maxdepth 1 -printf '%T@\\n' 2>/dev/null | " +
-		"awk 'BEGIN{m=0;n=0} {n++; if ($1>m) m=$1} END {printf \"%.0f %d\\n\", m, n-1}'"
-}
-
-func epochTime(s string) *time.Time {
-	epoch, err := strconv.ParseInt(s, 10, 64)
-	if err != nil || epoch <= 0 {
-		return nil
-	}
-	t := time.Unix(epoch, 0).UTC()
-	return &t
-}
-
-// measure is one item, measured now, by its own method.
-func (b *Backups) measure(ctx context.Context, it Item) Measured {
-	at := b.Now().UTC()
-	m := Measured{MeasuredAt: &at}
-	if !b.exists(ctx, it.Path) {
-		m.Error = it.Path + " does not exist"
-		zero := int64(0)
-		m.SizeBytes, m.Precision = &zero, "exact"
-		return m
-	}
-	m.Redundancy = b.redundancyOf(ctx, it.Path)
-	switch it.Measure {
-	case measureDataset, measureShallow:
-		out, err := b.Run(ctx, "bash", "-c", topCommand(it.Path))
-		if f := strings.Fields(out); err == nil && len(f) == 2 {
-			m.LastWrite = epochTime(f[0])
-		} else if err != nil {
-			m.Error = err.Error()
-		}
-		m.Precision = "no size: the newest change among its top-level entries only"
-		if it.Measure == measureShallow {
-			return m
-		}
-		out, err = b.Run(ctx, "zfs", "list", "-Hp", "-o", "name,used", it.Path)
-		f := strings.Fields(out)
-		if err != nil || len(f) != 2 {
-			m.Precision = "no size: it is on no ZFS dataset to read one from; the newest change among its top-level entries only"
-			return m
-		}
-		used, err := strconv.ParseInt(f[1], 10, 64)
-		if err != nil {
-			return m
-		}
-		m.SizeBytes = &used
-		m.Precision = "dataset " + f[0] + ": its whole size, from the filesystem's counters; the newest change among the top-level entries"
-		return m
-	}
-	out, err := b.Run(ctx, "bash", "-c", walkCommand(it.Path))
-	if err != nil {
-		m.Error = err.Error()
-		return m
-	}
-	lines := strings.Split(strings.TrimSpace(out), "\n")
-	f := strings.Fields(lines[0])
-	if len(f) != 3 || len(lines) < 2 {
-		m.Error = "the measurement said " + strings.TrimSpace(out)
-		return m
-	}
-	size, err1 := strconv.ParseInt(f[0], 10, 64)
-	files, err2 := strconv.Atoi(f[2])
-	if err1 != nil || err2 != nil {
-		m.Error = "the measurement said " + strings.TrimSpace(out)
-		return m
-	}
-	m.SizeBytes, m.LastWrite = &size, epochTime(f[1])
-	m.Precision = "exact"
-	if status := strings.TrimSpace(lines[len(lines)-1]); status == "124" || files >= walkFiles {
-		m.Precision = fmt.Sprintf("partial: measured partially — stopped after %s or %d files; the size is a lower bound, "+
-			"and this item wants measure: dataset or shallow", walkFor, walkFiles)
-	}
-	return m
-}
-
-// redundancyOf is the redundant storage a path is on, read as root: a ZFS pool's health and its own
-// verdict, an md array's members, a btrfs filesystem's error counters. Nil for storage with no
-// redundancy this holder knows how to read — which, for an item said to be protected by redundancy, the
-// controller says is no protection at all.
-func (b *Backups) redundancyOf(ctx context.Context, path string) *Redundancy {
-	out, err := b.Run(ctx, "findmnt", "-no", "SOURCE,FSTYPE", "--target", path)
-	if err != nil {
-		return nil
-	}
-	f := strings.Fields(out)
-	if len(f) < 2 {
-		return nil
-	}
-	source, fstype := f[0], f[1]
-	yes, no := true, false
-	switch {
-	case fstype == "zfs":
-		pool, _, _ := strings.Cut(source, "/")
-		r := &Redundancy{Kind: "zfs", Where: pool}
-		health, err := b.Run(ctx, "zpool", "list", "-H", "-o", "health", pool)
-		if err != nil {
-			r.Said = "zpool list: " + err.Error()
-			return r
-		}
-		status, err := b.Run(ctx, "zpool", "status", "-x", pool)
-		if err != nil {
-			r.Said = "zpool status: " + err.Error()
-			return r
-		}
-		health, status = strings.TrimSpace(health), strings.TrimSpace(status)
-		r.Said = "health " + health + "; " + firstLineOf(status)
-		if health == "ONLINE" && strings.Contains(status, "is healthy") {
-			r.Healthy = &yes
-		} else {
-			r.Healthy = &no
-			r.Said = "health " + health + "; " + oneLineOf(status)
-		}
-		return r
-	case strings.HasPrefix(source, "/dev/md"):
-		device := strings.TrimPrefix(source, "/dev/")
-		r := &Redundancy{Kind: "md", Where: device}
-		mdstat, err := b.Run(ctx, "cat", "/proc/mdstat")
-		if err != nil {
-			r.Said = err.Error()
-			return r
-		}
-		healthy, said, found := mdHealth(mdstat, device)
-		if !found {
-			r.Said = device + " is not in /proc/mdstat"
-			return r
-		}
-		r.Said = said
-		if healthy {
-			r.Healthy = &yes
-		} else {
-			r.Healthy = &no
-		}
-		return r
-	case fstype == "btrfs":
-		r := &Redundancy{Kind: "btrfs", Where: source}
-		stats, err := b.Run(ctx, "btrfs", "device", "stats", "--check", path)
-		if err != nil {
-			r.Healthy, r.Said = &no, "device errors: "+oneLineOf(err.Error())
-			return r
-		}
-		r.Healthy, r.Said = &yes, firstLineOf(stats)
-		return r
-	}
-	return nil
-}
-
-var mdMembers = regexp.MustCompile(`\[([U_]+)\]`)
-
-// mdHealth reads one array's block of /proc/mdstat: healthy when every member is up and it is not
-// recovering.
-func mdHealth(mdstat, device string) (bool, string, bool) {
-	lines := strings.Split(mdstat, "\n")
-	for i, l := range lines {
-		if !strings.HasPrefix(l, device+" ") {
-			continue
-		}
-		block := l
-		for j := i + 1; j < len(lines) && strings.HasPrefix(lines[j], " "); j++ {
-			block += " " + strings.TrimSpace(lines[j])
-		}
-		members := mdMembers.FindStringSubmatch(block)
-		healthy := members != nil && !strings.Contains(members[1], "_") && !strings.Contains(block, "recovery")
-		return healthy, oneLineOf(block), true
-	}
-	return false, "", false
-}
-
-func firstLineOf(s string) string {
-	line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
-	return line
-}
-
-func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") }
-
-// MeasureAll measures every item that is not a cache, one at a time, and keeps what it found. An item
-// measured by a walk is walked only when walks is true or its last walk is older than walkEvery: the
-// hourly round reads counters, and a walk runs at most once a day.
-func (b *Backups) MeasureAll(ctx context.Context, walks bool) error {
-	// The hourly round runs in its own process and the walk after a night in the night's (novox/hq issue 454).
-	release, err := b.hold(measureLock)
-	if err != nil {
-		return err
-	}
-	defer release()
-	items, err := b.Items()
-	if err != nil {
-		return err
-	}
-	before := b.Measurements()
-	found := map[string]map[string]Measured{}
-	for module, its := range items {
-		for _, it := range its {
-			if it.Class == classCache {
-				continue
-			}
-			if found[module] == nil {
-				found[module] = map[string]Measured{}
-			}
-			if was, ok := before[module][it.ID]; ok && it.Measure == measureWalk && !walks && was.MeasuredAt != nil &&
-				b.Now().Sub(*was.MeasuredAt) < walkEvery {
-				found[module][it.ID] = was
-				continue
-			}
-			found[module][it.ID] = b.measure(ctx, it)
-		}
-	}
-	raw, _ := json.MarshalIndent(found, "", "  ")
-	return os.WriteFile(b.measuredFile(), append(raw, '\n'), 0o600)
-}
-
 func (b *Backups) goodFile() string { return filepath.Join(b.Where.State, "good.json") }
 
 // Good is each module's newest good night: what a night that failed since does not erase.
@@ -418,14 +102,18 @@ func (b *Backups) recordGood(module string, at time.Time) {
 	if err := os.WriteFile(b.goodFile(), append(raw, '\n'), 0o600); err != nil {
 		b.Say("recording %s's good night failed: %v", module, err)
 	}
+	b.keepGood()
 }
 
-// itemReports is every item of one module, with its newest measurement and its newest good backup: the
+// itemReports is every item of one module, with its newest reading and its newest good backup: the
 // module's newest good night, where that night keeps the path that covers the item.
-func itemReports(its []Item, measured map[string]Measured, paths []string, good time.Time) []ItemReport {
+func itemReports(its []Item, measured map[string]health.Reading, paths []string, good time.Time) []ItemReport {
 	out := []ItemReport{}
 	for _, it := range its {
-		r := ItemReport{Item: it, Measured: measured[it.ID]}
+		r := ItemReport{Item: it}
+		if m, ok := measured[it.ID]; ok {
+			r.Measured, r.Peak, r.PeakDay = m.Measured, m.Peak, m.PeakAt
+		}
 		if it.CoveredBy != "" && !good.IsZero() {
 			for _, p := range paths {
 				if p == it.CoveredBy {

modules/restic/cmd/restic-backups/data_test.go

@@ -6,7 +6,8 @@ package main
 import (
 	"context"
 	"errors"
-	"fmt"
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	"git.novox.be/novox/mesh-sdk/go/health"
 	"os"
 	"path/filepath"
 	"strings"
@@ -28,53 +29,34 @@ func withData(t *testing.T, declared, data string) Where {
 }
 
 func TestTheComposedDataFileIsReadIntoEachModulesItems(t *testing.T) {
-	got, err := parseItems(composedData)
+	b := &Backups{Where: withData(t, composed, composedData), Now: time.Now, Say: quiet}
+	got, err := b.Items()
 	must(t, err)
 	if len(got["postgres"]) != 2 || got["postgres"][0].CoveredBy != "/var/lib/mesh-store/dumps" || got["postgres"][1].CoveredBy != "" ||
 		got["searxng"][0].Class != "cache" {
 		t.Fatalf("%+v", got)
 	}
-	for _, bad := range []string{"# pg\nitem a irreplaceable relative -\n", "# pg\nitem a irreplaceable /x\n", "# pg\nitem a b /x; rm -rf / -\n"} {
-		if _, err := parseItems(bad); err == nil || !strings.Contains(err.Error(), "pg declares a data line") {
-			t.Errorf("%q: %v", bad, err)
-		}
-	}
-	// An older controller composes no data file: nothing is measured, nothing fails.
-	b := &Backups{Where: placed(t, composed), Now: time.Now, Say: quiet}
+	// An older controller composes no data file: nothing is declared, nothing fails.
+	b = &Backups{Where: placed(t, composed), Now: time.Now, Say: quiet}
 	if items, err := b.Items(); err != nil || len(items) != 0 {
 		t.Fatalf("%v, %v", items, err)
 	}
 }
 
-// Every item but a cache is measured — its files' size and its newest change, in one walk as root — and
-// `backed-up` says each with the newest good night of the module that keeps the path covering it.
-func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T) {
+// `backed-up` says every item with disk-load's newest reading and its peak, and the newest good night of the module that
+// keeps the path covering it; an item with no reading yet is said with none.
+func TestEveryItemIsSaidWithItsReadingAndItsLastGoodBackup(t *testing.T) {
 	where := withData(t, composed, composedData)
-	var mu sync.Mutex
-	var walked []string
-	run := func(_ context.Context, name string, args ...string) (string, error) {
-		mu.Lock()
-		defer mu.Unlock()
-		switch {
-		case name == "test":
-			return "", nil
-		case name == "bash" && strings.Contains(args[1], "find '"):
-			walked = append(walked, args[1])
-			if strings.Contains(args[1], "'/var/lib/mesh-store'") {
-				return "1073741824 1759744800 4000\n0\n", nil
-			}
-			return "5000 1759700000 3\n0\n", nil
-		case name == "restic":
-			return "[]", nil
-		}
-		return "", nil
-	}
+	size, peak := int64(1<<30), int64(2<<30)
+	at := time.Date(2026, 10, 6, 4, 0, 0, 0, time.UTC)
+	write := time.Unix(1759744800, 0).UTC()
 	night := time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC)
-	b := &Backups{Where: where, Run: run, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet}
-	must(t, b.MeasureAll(context.Background(), true))
-	if len(walked) != 2 {
-		t.Fatalf("walked %d paths, want the two that are not a cache: %v", len(walked), walked)
-	}
+	b := &Backups{Where: where, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet,
+		Run: func(context.Context, string, ...string) (string, error) { return "[]", nil },
+		Readings: func() ([]health.Reading, error) {
+			return []health.Reading{{V: 1, Node: "ace", PlacedDataItem: catalogue.PlacedDataItem{Module: "postgres", ID: "store"},
+				Measured: health.Measured{SizeBytes: &size, LastWrite: &write, MeasuredAt: &at, Precision: "exact"}, Peak: &peak, PeakAt: "2026-10-05"}}, nil
+		}}
 	b.recordGood("postgres", night)
 	got, err := b.BackedUp(context.Background(), "")
 	must(t, err)
@@ -89,12 +71,23 @@ func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T)
 			}
 		}
 	}
-	if store == nil || store.SizeBytes == nil || *store.SizeBytes != 1<<30 || store.LastWrite == nil ||
-		store.LastWrite.Unix() != 1759744800 || store.LastBackup == nil || !store.LastBackup.Equal(night) {
+	if store == nil || store.SizeBytes == nil || *store.SizeBytes != 1<<30 || store.Peak == nil || *store.Peak != 2<<30 ||
+		store.LastWrite == nil || store.LastWrite.Unix() != 1759744800 || store.LastBackup == nil || !store.LastBackup.Equal(night) {
 		t.Fatalf("the store is said as %+v", store)
 	}
-	if dumps == nil || dumps.LastBackup != nil {
-		t.Fatalf("an item not backed up is said backed up: %+v", dumps)
+	if dumps == nil || dumps.LastBackup != nil || dumps.SizeBytes != nil {
+		t.Fatalf("an item with no reading or no backup is said with one: %+v", dumps)
+	}
+	// The same last good night is kept on the bus beside good.json, one row per item a backup path covers.
+	b.Node = "ace"
+	wrote := map[string]health.GoodRow{}
+	b.PutGood = func(k string, r health.GoodRow) error { wrote[k] = r; return nil }
+	b.recordGood("postgres", night) // a good night writes the rows, once; the same night again writes nothing
+	n := len(wrote)
+	b.recordGood("postgres", night)
+	row, ok := wrote[health.GoodKey("ace", "postgres", "store")]
+	if len(wrote) != 1 || n != 1 || !ok || !row.LastGood.Equal(night) || row.CoveredBy != "/var/lib/mesh-store/dumps" || row.V != health.GoodVersion {
+		t.Fatalf("the good rows are %+v", wrote)
 	}
 }
 
@@ -202,104 +195,3 @@ func TestARetiredItemIsDeletedOnlyAfterALastRestorePoint(t *testing.T) {
 		t.Fatalf("%+v, %v", again, err)
 	}
 }
-
-// The redundant storage under an item is read: a ZFS pool healthy, then degraded; an md array with a
-// member missing; and plain storage, which is no redundancy at all.
-func TestTheRedundantStorageUnderAnItemIsRead(t *testing.T) {
-	health, statusX := "ONLINE\n", "pool 'storage' is healthy\n"
-	fs := "storage/media zfs\n"
-	run := func(_ context.Context, name string, args ...string) (string, error) {
-		switch name {
-		case "findmnt":
-			return fs, nil
-		case "zpool":
-			if args[0] == "list" {
-				return health, nil
-			}
-			return statusX, nil
-		case "cat":
-			return "Personalities : [raid1]\nmd127 : active raid1 sdb1[1] sda1[0]\n      976630464 blocks super 1.2 [2/1] [U_]\n\nunused devices: <none>\n", nil
-		}
-		return "", nil
-	}
-	b := &Backups{Run: run, Now: time.Now, Say: quiet}
-	r := b.redundancyOf(context.Background(), "/storage/media/movies")
-	if r == nil || r.Kind != "zfs" || r.Where != "storage" || r.Healthy == nil || !*r.Healthy {
-		t.Fatalf("a healthy pool: %+v", r)
-	}
-	health, statusX = "DEGRADED\n", "  pool: storage\n state: DEGRADED\nstatus: One or more devices has been removed\n"
-	if r := b.redundancyOf(context.Background(), "/storage/media/movies"); r.Healthy == nil || *r.Healthy || !strings.Contains(r.Said, "DEGRADED") {
-		t.Fatalf("a degraded pool: %+v", r)
-	}
-	fs = "/dev/md127 ext4\n"
-	if r := b.redundancyOf(context.Background(), "/srv/x"); r == nil || r.Kind != "md" || r.Healthy == nil || *r.Healthy {
-		t.Fatalf("an array missing a member: %+v", r)
-	}
-	fs = "/dev/nvme0n1p3 ext4\n"
-	if r := b.redundancyOf(context.Background(), "/var/lib/x"); r != nil {
-		t.Fatalf("plain storage read as redundant: %+v", r)
-	}
-}
-
-// Large items are never walked: a dataset is measured from the filesystem's counters and its top-level
-// entries, a shallow item from its top-level entries only; a walk that meets more than its bound stops
-// and says so; and a walk runs at most once a day — the hourly round keeps the last one.
-func TestNothingLargeIsWalkedAndAWalkIsBoundedAndDaily(t *testing.T) {
-	data := "# media\nitem movies irreplaceable /storage/media/movies - redundancy dataset\n" +
-		"item meta rebuildable /mnt/plex/config /mnt/plex/config backup shallow\n" +
-		"item big valuable /srv/big /srv/big backup\n"
-	where := withData(t, composed, data)
-	var mu sync.Mutex
-	var walks []string
-	truncated := true
-	run := func(_ context.Context, name string, args ...string) (string, error) {
-		mu.Lock()
-		defer mu.Unlock()
-		line := name + " " + strings.Join(args, " ")
-		switch {
-		case name == "test":
-			return "", nil
-		case name == "zfs" && args[len(args)-1] == "/storage/media/movies":
-			return "storage/media\t97067690722560\n", nil
-		case name == "zfs":
-			return "", errors.New("not a ZFS dataset")
-		case name == "bash" && strings.Contains(line, "-maxdepth 1"):
-			return "1759744800 12673\n", nil
-		case name == "bash":
-			walks = append(walks, line)
-			if strings.Contains(line, "-xdev") && !strings.Contains(line, "'/srv/big'") {
-				t.Errorf("walked something declared not to be walked: %s", line)
-			}
-			if truncated {
-				return fmt.Sprintf("123 1759700000 %d\n141\n", walkFiles), nil
-			}
-			return "999 1759700000 10\n0\n", nil
-		}
-		return "", nil
-	}
-	now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
-	b := &Backups{Where: where, Run: run, Now: func() time.Time { return now }, Say: quiet}
-	must(t, b.MeasureAll(context.Background(), false))
-	got := b.Measurements()["media"]
-	if m := got["movies"]; m.SizeBytes == nil || *m.SizeBytes != 97067690722560 || !strings.HasPrefix(m.Precision, "dataset storage/media") || m.LastWrite == nil {
-		t.Fatalf("the library from its dataset: %+v", m)
-	}
-	if m := got["meta"]; m.SizeBytes != nil || !strings.HasPrefix(m.Precision, "no size") || m.LastWrite == nil {
-		t.Fatalf("a shallow item: %+v", m)
-	}
-	if m := got["big"]; !strings.HasPrefix(m.Precision, "partial") {
-		t.Fatalf("a walk stopped at its bound: %+v", m)
-	}
-	// An hour later the hourly round reads counters again and walks nothing.
-	now = now.Add(time.Hour)
-	truncated = false
-	must(t, b.MeasureAll(context.Background(), false))
-	if len(walks) != 1 {
-		t.Fatalf("walked %d times in two rounds an hour apart: %v", len(walks), walks)
-	}
-	// After the night, it walks.
-	must(t, b.MeasureAll(context.Background(), true))
-	if len(walks) != 2 || b.Measurements()["media"]["big"].Precision != "exact" {
-		t.Fatalf("%d walks, %+v", len(walks), b.Measurements()["media"]["big"])
-	}
-}

modules/restic/cmd/restic-backups/good.go

@@ -0,0 +1,128 @@
+package main
+
+// **Each item's last good night, on the bus** (novox/hq ADR 0571 clause 3, ADR 0564 §6): beside good.json, the module's
+// own state `good` (health.GoodRow), one key `<node>.<module>.<item>`, written when a night's files change, so the
+// self-check's probe of a stale backup reads it there.
+
+import (
+	"encoding/json"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+	"unsafe"
+
+	"git.novox.be/novox/mesh-sdk/go/health"
+)
+
+// GoodRows is the rows of this machine (health.GoodRow): every item a module declares that a path of the module's backup
+// covers, with the module's newest good night.
+func (b *Backups) GoodRows() (map[string]health.GoodRow, error) {
+	items, err := b.Items()
+	if err != nil {
+		return nil, err
+	}
+	declared, err := b.Declared()
+	if err != nil {
+		return nil, err
+	}
+	paths := map[string][]string{}
+	for _, d := range declared {
+		paths[d.Module] = d.Paths
+	}
+	good := b.Good()
+	out := map[string]health.GoodRow{}
+	for module, its := range items {
+		for _, it := range its {
+			if it.Class == "cache" || it.CoveredBy == "" {
+				continue
+			}
+			covered := false
+			for _, p := range paths[module] {
+				covered = covered || p == it.CoveredBy
+			}
+			if !covered {
+				continue
+			}
+			out[health.GoodKey(b.Node, module, it.ID)] = health.GoodRow{V: health.GoodVersion, Node: b.Node, Module: module,
+				Item: it.ID, CoveredBy: it.CoveredBy, LastGood: good[module].UTC()}
+		}
+	}
+	return out, nil
+}
+
+// keepGood writes the rows that differ from what was last written, when a night succeeded (recordGood). A bus that
+// cannot be reached loses nothing: good.json is the local copy, and the next good night writes the rows again.
+func (b *Backups) keepGood() {
+	if b.PutGood == nil {
+		return
+	}
+	rows, err := b.GoodRows()
+	if err != nil {
+		b.Say("the last good nights could not be read: %v", err)
+		return
+	}
+	b.goodMu.Lock()
+	defer b.goodMu.Unlock()
+	if b.goodWrote == nil {
+		b.goodWrote = map[string][]byte{}
+	}
+	for k, r := range rows {
+		raw, _ := json.Marshal(r)
+		if string(b.goodWrote[k]) == string(raw) {
+			continue
+		}
+		if err := b.PutGood(k, r); err != nil {
+			b.Say("the last good night of %s was not put on the bus: %v", k, err)
+			continue
+		}
+		b.goodWrote[k] = raw
+	}
+}
+
+// watchGood writes the rows once at start and then each time the night's process changes what it keeps in the state
+// directory (good.json, the nights' file), as the file system says it: the night runs as the operator account, which a
+// process with a bus account of its own may not (ADR 0436 §2), so the tools' runtime, which has the module's account,
+// writes at the night's event instead of the night itself. No timer.
+func watchGood(b *Backups) {
+	time.Sleep(20 * time.Second) // the runtime serves first
+	b.keepGood()
+	for {
+		err := whenWritten(b.Where.State, func(name string) bool { return name == "good.json" || name == filepath.Base(b.nightsFile()) }, b.keepGood)
+		b.Say("watching the state directory for a night failed (%v); trying again in a minute", err)
+		time.Sleep(time.Minute)
+		b.keepGood()
+	}
+}
+
+// whenWritten calls then each time a file of dir that match names is written or put in place, until the watch fails.
+func whenWritten(dir string, match func(name string) bool, then func()) error {
+	fd, err := syscall.InotifyInit1(syscall.IN_CLOEXEC)
+	if err != nil {
+		return err
+	}
+	defer syscall.Close(fd)
+	if _, err := syscall.InotifyAddWatch(fd, dir, syscall.IN_CLOSE_WRITE|syscall.IN_MOVED_TO); err != nil {
+		return err
+	}
+	buf := make([]byte, 64*syscall.SizeofInotifyEvent+4096)
+	for {
+		n, err := syscall.Read(fd, buf)
+		if err != nil {
+			if err == syscall.EINTR {
+				continue
+			}
+			return err
+		}
+		heard := false
+		for off := 0; off+syscall.SizeofInotifyEvent <= n; {
+			ev := (*syscall.InotifyEvent)(unsafe.Pointer(&buf[off]))
+			name := strings.TrimRight(string(buf[off+syscall.SizeofInotifyEvent:off+syscall.SizeofInotifyEvent+int(ev.Len)]), "\x00")
+			heard = heard || match(name)
+			off += syscall.SizeofInotifyEvent + int(ev.Len)
+		}
+		if heard {
+			then()
+		}
+	}
+}

modules/restic/cmd/restic-backups/health_test.go

@@ -263,8 +263,8 @@ func TestTheManifestRunsTheNightJudgedByItsHealthTool(t *testing.T) {
 	}
 	env := m.Build.Artifacts[0].Env
 	want := map[string]struct{ run, kind, tool string }{
-		"night":   {"night", "tool", "backup_health"},
-		"measure": {"measure", "unit", ""},
+		"night": {"night", "unit", ""},
+		"awake": {"awake", "tool", "backup_health"},
 	}
 	for _, r := range m.Resources {
 		w, ok := want[r["id"].(string)]

modules/restic/cmd/restic-backups/main.go

@@ -24,6 +24,8 @@ import (
 	"time"
 
 	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/announce"
+	"git.novox.be/novox/mesh-sdk/go/health"
 )
 
 // Seat is the role this module holds.
@@ -46,20 +48,29 @@ func main() {
 	}
 	if way == "" {
 		b.Emit = func(event string, body any) error { return stdio.Emit(event, body) }
+		b.Readings = busReadings(b.Node)
 	}
 	switch way {
 	case "night":
 		nights(b)
-	case "measure":
-		measurements(b)
+	case "awake":
+		counting(b)
 	case "":
+		if err := stdio.Keeps(announce.State{Name: health.GoodState}); err != nil {
+			say("%v", err)
+			os.Exit(1)
+		}
+		b.PutGood = func(key string, row health.GoodRow) error {
+			_, err := stdio.State(health.GoodState).Put(key, row)
+			return err
+		}
+		go watchGood(b)
 		if err := stdio.Serve("", tools(b)); err != nil {
 			say("%v", err)
 			os.Exit(1)
 		}
 	default:
-		say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights, with "+
-			"measure it measures the data declared here", way)
+		say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights, with awake it counts how long the machine is up since a night", way)
 		os.Exit(2)
 	}
 }
@@ -72,7 +83,6 @@ func nights(b *Backups) {
 	if h, err := strconv.Atoi(os.Getenv("MESH_BACKUP_HOUR")); err == nil && h >= 0 && h < 24 {
 		hour = h
 	}
-	go counting(b)
 	if missedANight(b.Nights(), time.Now()) {
 		time.Sleep(10 * time.Minute)
 		night(b)
@@ -83,23 +93,6 @@ func nights(b *Backups) {
 	}
 }
 
-// measurements measures every declared item once an hour (MESH_BACKUP_MEASURE_EVERY to change it),
-// the first a few minutes after start — counters hourly, a walk at most daily — and every walk after
-// each night (novox/hq ADR 0233).
-func measurements(b *Backups) {
-	every := time.Hour
-	if d, err := time.ParseDuration(os.Getenv("MESH_BACKUP_MEASURE_EVERY")); err == nil && d >= time.Minute {
-		every = d
-	}
-	time.Sleep(3 * time.Minute)
-	for {
-		if err := b.MeasureAll(context.Background(), false); err != nil {
-			say("measuring the data declared here failed: %v", err)
-		}
-		time.Sleep(every)
-	}
-}
-
 func night(b *Backups) {
 	ctx := context.Background()
 	outcome, err := b.BackUp(ctx, "")
@@ -116,9 +109,6 @@ func night(b *Backups) {
 	if len(failed) > 0 {
 		say("the night left %s without a backup", strings.Join(failed, ", "))
 	}
-	if err := b.MeasureAll(ctx, true); err != nil {
-		say("measuring the data declared here failed: %v", err)
-	}
 	// Sundays, the repository's own integrity with a sample of the data read back.
 	if time.Now().Weekday() == time.Sunday {
 		if err := b.Check(ctx); err != nil {
@@ -204,3 +194,28 @@ func tools(b *Backups) []stdio.Tool {
 			Run:   func(a map[string]any) (any, error) { return b.DeletedOutcome(arg(a, "path")) }},
 	}
 }
+
+// busReadings reads disk-load's readings of this machine from its state, every key `<node>.<module>.<item>`.
+func busReadings(node string) func() ([]health.Reading, error) {
+	return func() ([]health.Reading, error) {
+		st := stdio.State(health.ReadingsRead)
+		keys, err := st.Keys()
+		if err != nil {
+			return nil, err
+		}
+		var out []health.Reading
+		for _, k := range keys {
+			if !strings.HasPrefix(k, node+".") {
+				continue
+			}
+			e, err := st.Get(k)
+			if err != nil || e == nil {
+				continue
+			}
+			if r, err := health.DecodeReading(e.Value); err == nil {
+				out = append(out, r)
+			}
+		}
+		return out, nil
+	}
+}