Design 47's last milestones: the keeper, self-check, healers, hand-acts, provisioner, vault, artifact sweep, disk readings, credential forms (issues 774, 1952, 1953, 1954, 1955)

One commit with exactly the tree of the reviewed head 35b4509c0a, on main 12c2574c9d, so two compiled programs added and removed on the branch leave no trace in history (issue 1577).

modules/disk-load/cmd/disk-load/keeper.go

@@ -0,0 +1,248 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	"git.novox.be/novox/mesh-sdk/go/health"
+	"os"
+	"os/exec"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	stdio "git.novox.be/novox/mesh-sdk/go"
+)
+
+// Store is the state `readings` as the keeper uses it.
+type Store interface {
+	Get(key string) (json.RawMessage, error)
+	Put(key string, value any) error
+	Keys() ([]string, error)
+}
+
+// busStore is the state through the runtime.
+type busStore struct{}
+
+func (busStore) Get(key string) (json.RawMessage, error) {
+	e, err := stdio.State(StateReadings).Get(key)
+	if err != nil || e == nil {
+		return nil, err
+	}
+	return e.Value, nil
+}
+func (busStore) Put(key string, v any) error {
+	_, err := stdio.State(StateReadings).Put(key, v)
+	return err
+}
+func (busStore) Keys() ([]string, error) { return stdio.State(StateReadings).Keys() }
+
+// Keeper measures the items of its node and keeps the readings.
+type Keeper struct {
+	Node string
+	// Items is every data item the modules placed on this node declare, composed from the bus (frombus.DataItems).
+	Items    func(ctx context.Context) (map[string][]catalogue.PlacedDataItem, error)
+	Measurer Measurer
+	Store    Store
+	Say      func(string, ...any)
+
+	mu sync.Mutex // one round at a time
+}
+
+// Round measures every item that is not a cache, one at a time, and writes the readings that changed. An item
+// measured by a walk is walked only when walks is true or its last walk is older than WalkEvery: the hourly round
+// reads counters, and a walk runs at most once a day (ADR 0233 §22).
+func (k *Keeper) Round(ctx context.Context, walks bool) (written, kept int, err error) {
+	k.mu.Lock()
+	defer k.mu.Unlock()
+	items, err := k.Items(ctx)
+	if err != nil {
+		return 0, 0, err
+	}
+	modules := make([]string, 0, len(items))
+	for m := range items {
+		modules = append(modules, m)
+	}
+	sort.Strings(modules)
+	var failed []string
+	for _, module := range modules {
+		for _, it := range items[module] {
+			if it.Class == ClassCache {
+				continue
+			}
+			w, err := k.measureOne(ctx, it, walks)
+			if err != nil {
+				failed = append(failed, fmt.Sprintf("%s.%s: %v", module, it.ID, err))
+				continue
+			}
+			if w {
+				written++
+			} else {
+				kept++
+			}
+		}
+	}
+	retired, err := k.retireGone(items)
+	written += retired
+	if err != nil {
+		failed = append(failed, err.Error())
+	}
+	if len(failed) > 0 {
+		return written, kept, errors.New("readings not kept: " + strings.Join(failed, "; "))
+	}
+	return written, kept, nil
+}
+
+// retireGone marks retired every reading of this node whose item the node no longer declares — its module was unassigned
+// there, or is no longer pulled in — with when and why, and keeps it: the record of what the module left behind, until a
+// person deletes it (ADR 0233). It is cleared when the item is declared again (measureOne).
+func (k *Keeper) retireGone(declared map[string][]catalogue.PlacedDataItem) (int, error) {
+	keys, err := k.Store.Keys()
+	if err != nil {
+		return 0, err
+	}
+	declaredNow := map[string]bool{}
+	for module, its := range declared {
+		for _, it := range its {
+			declaredNow[health.ReadingKey(k.Node, module, it.ID)] = true
+		}
+	}
+	now := k.Measurer.Now().UTC()
+	n := 0
+	for _, key := range keys {
+		if !strings.HasPrefix(key, k.Node+".") || declaredNow[key] {
+			continue
+		}
+		raw, err := k.Store.Get(key)
+		if err != nil || len(raw) == 0 {
+			continue
+		}
+		r, err := health.DecodeReading(raw)
+		if err != nil || r.Node != k.Node || !r.RetiredAt.IsZero() {
+			continue
+		}
+		r.RetiredAt = now
+		r.RetiredWhy = fmt.Sprintf("no longer declared on %s: its module was unassigned there, or is no longer pulled in", k.Node)
+		if err := k.Store.Put(key, r); err != nil {
+			return n, err
+		}
+		n++
+	}
+	return n, nil
+}
+
+func (k *Keeper) measureOne(ctx context.Context, it catalogue.PlacedDataItem, walks bool) (bool, error) {
+	key := health.ReadingKey(k.Node, it.Module, it.ID)
+	var prev *health.Reading
+	raw, err := k.Store.Get(key)
+	if err != nil {
+		return false, err
+	}
+	if len(raw) > 0 {
+		if p, err := health.DecodeReading(raw); err == nil {
+			prev = &p
+		}
+	}
+	now := k.Measurer.Now()
+	if prev != nil && prev.RetiredAt.IsZero() && it.Measure == catalogue.MeasureWalk && !walks && prev.MeasuredAt != nil && now.Sub(*prev.MeasuredAt) < WalkEvery {
+		return false, nil
+	}
+	m := k.Measurer.Measure(ctx, it)
+	next, changed := health.NextReading(prev, k.Node, it, m, now)
+	if prev != nil && !prev.RetiredAt.IsZero() {
+		// Declared again: it is no longer retired.
+		next.RetiredAt, next.RetiredWhy, changed = time.Time{}, "", true
+	}
+	if !changed {
+		return false, nil
+	}
+	return true, k.Store.Put(key, next)
+}
+
+// MeasureNow measures one item now, walk or not, and writes its reading when it changed.
+func (k *Keeper) MeasureNow(ctx context.Context, module, item string) (health.Reading, bool, error) {
+	k.mu.Lock()
+	defer k.mu.Unlock()
+	items, err := k.Items(ctx)
+	if err != nil {
+		return health.Reading{}, false, err
+	}
+	for _, it := range items[module] {
+		if it.ID != item {
+			continue
+		}
+		if it.Class == ClassCache {
+			return health.Reading{}, false, fmt.Errorf("%s.%s is a cache, which is listed and never measured", module, item)
+		}
+		w, err := k.measureOne(ctx, it, true)
+		if err != nil {
+			return health.Reading{}, false, err
+		}
+		raw, err := k.Store.Get(health.ReadingKey(k.Node, module, item))
+		if err != nil {
+			return health.Reading{}, w, err
+		}
+		r, err := health.DecodeReading(raw)
+		return r, w, err
+	}
+	return health.Reading{}, false, fmt.Errorf("%s declares no data item %q on this machine", module, item)
+}
+
+// Loop runs a round after a few minutes and then every `every`, with the walks of the night's round when it is the
+// hour after the night; ctx ends it.
+func (k *Keeper) Loop(ctx context.Context, every time.Duration, first time.Duration) {
+	select {
+	case <-ctx.Done():
+		return
+	case <-time.After(first):
+	}
+	for {
+		w, kept, err := k.Round(ctx, false)
+		if err != nil {
+			k.Say("measuring the data declared here: %v", err)
+		} else {
+			k.Say("measured the data declared here: %d readings changed, %d unchanged", w, kept)
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(every):
+		}
+	}
+}
+
+// RootRunner runs a command as root, through sudo without a prompt when this process is not root. A walk is bounded
+// by WalkFor itself; the runner's own bound is a little longer.
+func RootRunner(ctx context.Context, name string, args ...string) (string, error) {
+	ctx, cancel := context.WithTimeout(ctx, WalkFor+time.Minute)
+	defer cancel()
+	program, argv := name, args
+	if os.Getuid() != 0 {
+		program, argv = "sudo", append([]string{"-n", name}, args...)
+	}
+	var stdout, stderr bytes.Buffer
+	cmd := exec.CommandContext(ctx, program, argv...)
+	cmd.Stdout, cmd.Stderr = &stdout, &stderr
+	if err := cmd.Run(); err != nil {
+		said := strings.TrimSpace(stderr.String())
+		if said == "" {
+			said = strings.TrimSpace(stdout.String())
+		}
+		if program == "sudo" && strings.HasPrefix(said, "sudo:") {
+			return "", fmt.Errorf("%s needs root and the runtime's account may not run it without a prompt: %s", name, said)
+		}
+		lines := strings.Split(said, "\n")
+		if len(lines) > 3 {
+			lines = lines[len(lines)-3:]
+		}
+		if said == "" {
+			return "", fmt.Errorf("%s: %w", name, err)
+		}
+		return "", errors.New(strings.Join(lines, " / "))
+	}
+	return stdout.String(), nil
+}

modules/disk-load/cmd/disk-load/main.go

@@ -1,20 +1,61 @@
 // The disk-load module's Go bundle (novox/hq ADR 0188, ADR 0193, issue 312): a process the node's
 // runtime launches and speaks MCP over stdio to. It reads the machine's storage load from /proc and
 // /sys — iowait, pressure stall, each disk's busy share, throughput, IOPS, wait and queue depth, the
-// filesystems' space and the processes doing the I/O — and changes nothing.
+// filesystems' space and the processes doing the I/O — and changes nothing of the machine.
+//
+// It also measures the data items the modules of its node declare (ADR 0233) and keeps each reading and the item's
+// seven-day peak as its own state `readings` (ADR 0571 clause 3), composed from the bus (no file).
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
+	"strings"
+	"time"
 
 	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/announce"
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	"git.novox.be/novox/mesh-sdk/go/compose/frombus"
+	"git.novox.be/novox/mesh-sdk/go/compose/fromruntime"
 )
 
+func logf(format string, args ...any) { fmt.Fprintf(os.Stderr, "[disk-load] "+format+"\n", args...) }
+
 func main() {
+	if err := stdio.Keeps(announce.State{Name: StateReadings}); err != nil {
+		logf("%v", err)
+		os.Exit(1)
+	}
+	node := os.Getenv("MESH_NODE")
+	k := &Keeper{Node: node, Store: busStore{}, Say: logf, Measurer: Measurer{Run: RootRunner, Now: time.Now},
+		Items: func(ctx context.Context) (map[string][]catalogue.PlacedDataItem, error) {
+			return placedItems(ctx, node)
+		}}
+	if node != "" {
+		every := time.Hour
+		if d, err := time.ParseDuration(os.Getenv("MESH_DISK_MEASURE_EVERY")); err == nil && d >= time.Minute {
+			every = d
+		}
+		go k.Loop(context.Background(), every, 3*time.Minute)
+	}
+	tools := append(Tools(Here()), ReadingTools(k)...)
 	// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): disk-load.
-	if err := stdio.Serve("", Tools(Here())); err != nil {
+	if err := stdio.Serve("", tools); err != nil {
 		fmt.Fprintln(os.Stderr, err)
 		os.Exit(1)
 	}
 }
+
+// placedItems is every data item the modules placed on node declare, composed from the bus as the node composes itself.
+func placedItems(ctx context.Context, node string) (map[string][]catalogue.PlacedDataItem, error) {
+	items, unplaced, err := frombus.DataItems(ctx, fromruntime.Reader(), frombus.Mesh{}, frombus.Scope{Node: node})
+	if err != nil {
+		return nil, err
+	}
+	if len(unplaced) > 0 {
+		logf("data items left out as not placeable: %s", strings.Join(unplaced, "; "))
+	}
+	return items, nil
+}

modules/disk-load/cmd/disk-load/measure.go

@@ -0,0 +1,215 @@
+package main
+
+// How an item is measured (novox/hq ADR 0233 §22 to §25), moved here from the backup holder, which only reads the
+// readings now (ADR 0571 clause 3). Never an unbounded walk of something large.
+
+import (
+	"context"
+	"fmt"
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	"git.novox.be/novox/mesh-sdk/go/health"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// The bounds on a walk: a walk is for small items, and one that meets a large one stops and says so.
+var (
+	WalkFor   = 10 * time.Minute
+	WalkFiles = 2_000_000
+	WalkEvery = 23 * time.Hour
+)
+
+// walkCommand sums the files under a path and finds its newest change, in one walk, bounded by time and by files.
+// One line out: "<bytes> <epoch seconds> <files>", then the status of find.
+func walkCommand(path string) string {
+	return fmt.Sprintf("timeout %d find '%s' -xdev -printf '%%y %%s %%T@\\n' 2>/dev/null | head -n %d | "+
+		"awk 'BEGIN{s=0;m=0;n=0} {n++; if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%%d %%.0f %%d\\n\", s, m, n}'; "+
+		"echo \"${PIPESTATUS[0]:-0}\"", int(WalkFor.Seconds()), path, WalkFiles)
+}
+
+// topCommand is the newest change among a path and its top-level entries, and how many there are.
+func topCommand(path string) string {
+	return "timeout 60 find '" + path + "' -maxdepth 1 -printf '%T@\\n' 2>/dev/null | " +
+		"awk 'BEGIN{m=0;n=0} {n++; if ($1>m) m=$1} END {printf \"%.0f %d\\n\", m, n-1}'"
+}
+
+func epochTime(s string) *time.Time {
+	epoch, err := strconv.ParseInt(s, 10, 64)
+	if err != nil || epoch <= 0 {
+		return nil
+	}
+	t := time.Unix(epoch, 0).UTC()
+	return &t
+}
+
+// Measurer measures items through a runner that has the root the walks need.
+type Measurer struct {
+	Run Runner
+	Now func() time.Time
+}
+
+func (b Measurer) exists(ctx context.Context, path string) bool {
+	_, err := b.Run(ctx, "test", "-e", path)
+	return err == nil
+}
+
+// Measure is one item, measured now, by its own method.
+func (b Measurer) Measure(ctx context.Context, it catalogue.PlacedDataItem) health.Measured {
+	at := b.Now().UTC()
+	m := health.Measured{MeasuredAt: &at}
+	if !b.exists(ctx, it.Path) {
+		m.Error = it.Path + " does not exist"
+		zero := int64(0)
+		m.SizeBytes, m.Precision = &zero, "exact"
+		return m
+	}
+	m.Redundancy = b.redundancyOf(ctx, it.Path)
+	switch it.Measure {
+	case catalogue.MeasureDataset, catalogue.MeasureShallow:
+		out, err := b.Run(ctx, "bash", "-c", topCommand(it.Path))
+		if f := strings.Fields(out); err == nil && len(f) == 2 {
+			m.LastWrite = epochTime(f[0])
+		} else if err != nil {
+			m.Error = err.Error()
+		}
+		m.Precision = "no size: the newest change among its top-level entries only"
+		if it.Measure == catalogue.MeasureShallow {
+			return m
+		}
+		out, err = b.Run(ctx, "zfs", "list", "-Hp", "-o", "name,used", it.Path)
+		f := strings.Fields(out)
+		if err != nil || len(f) != 2 {
+			m.Precision = "no size: it is on no ZFS dataset to read one from; the newest change among its top-level entries only"
+			return m
+		}
+		used, err := strconv.ParseInt(f[1], 10, 64)
+		if err != nil {
+			return m
+		}
+		m.SizeBytes = &used
+		m.Precision = "dataset " + f[0] + ": its whole size, from the filesystem's counters; the newest change among the top-level entries"
+		return m
+	}
+	out, err := b.Run(ctx, "bash", "-c", walkCommand(it.Path))
+	if err != nil {
+		m.Error = err.Error()
+		return m
+	}
+	lines := strings.Split(strings.TrimSpace(out), "\n")
+	f := strings.Fields(lines[0])
+	if len(f) != 3 || len(lines) < 2 {
+		m.Error = "the measurement said " + strings.TrimSpace(out)
+		return m
+	}
+	size, err1 := strconv.ParseInt(f[0], 10, 64)
+	files, err2 := strconv.Atoi(f[2])
+	if err1 != nil || err2 != nil {
+		m.Error = "the measurement said " + strings.TrimSpace(out)
+		return m
+	}
+	m.SizeBytes, m.LastWrite = &size, epochTime(f[1])
+	m.Precision = "exact"
+	if status := strings.TrimSpace(lines[len(lines)-1]); status == "124" || files >= WalkFiles {
+		m.Precision = fmt.Sprintf("partial: measured partially — stopped after %s or %d files; the size is a lower bound, "+
+			"and this item wants measure: dataset or shallow", WalkFor, WalkFiles)
+	}
+	return m
+}
+
+// redundancyOf is the redundant storage a path is on: a ZFS pool's health, an md array's members, a btrfs
+// filesystem's error counters. Nil for storage with no redundancy it knows how to read.
+func (b Measurer) redundancyOf(ctx context.Context, path string) *health.Redundancy {
+	out, err := b.Run(ctx, "findmnt", "-no", "SOURCE,FSTYPE", "--target", path)
+	if err != nil {
+		return nil
+	}
+	f := strings.Fields(out)
+	if len(f) < 2 {
+		return nil
+	}
+	source, fstype := f[0], f[1]
+	yes, no := true, false
+	switch {
+	case fstype == "zfs":
+		pool, _, _ := strings.Cut(source, "/")
+		r := &health.Redundancy{Kind: "zfs", Where: pool}
+		health, err := b.Run(ctx, "zpool", "list", "-H", "-o", "health", pool)
+		if err != nil {
+			r.Said = "zpool list: " + err.Error()
+			return r
+		}
+		status, err := b.Run(ctx, "zpool", "status", "-x", pool)
+		if err != nil {
+			r.Said = "zpool status: " + err.Error()
+			return r
+		}
+		health, status = strings.TrimSpace(health), strings.TrimSpace(status)
+		r.Said = "health " + health + "; " + firstLineOf(status)
+		if health == "ONLINE" && strings.Contains(status, "is healthy") {
+			r.Healthy = &yes
+		} else {
+			r.Healthy = &no
+			r.Said = "health " + health + "; " + oneLineOf(status)
+		}
+		return r
+	case strings.HasPrefix(source, "/dev/md"):
+		device := strings.TrimPrefix(source, "/dev/")
+		r := &health.Redundancy{Kind: "md", Where: device}
+		mdstat, err := b.Run(ctx, "cat", "/proc/mdstat")
+		if err != nil {
+			r.Said = err.Error()
+			return r
+		}
+		healthy, said, found := mdHealth(mdstat, device)
+		if !found {
+			r.Said = device + " is not in /proc/mdstat"
+			return r
+		}
+		r.Said = said
+		if healthy {
+			r.Healthy = &yes
+		} else {
+			r.Healthy = &no
+		}
+		return r
+	case fstype == "btrfs":
+		r := &health.Redundancy{Kind: "btrfs", Where: source}
+		stats, err := b.Run(ctx, "btrfs", "device", "stats", "--check", path)
+		if err != nil {
+			r.Healthy, r.Said = &no, "device errors: "+oneLineOf(err.Error())
+			return r
+		}
+		r.Healthy, r.Said = &yes, firstLineOf(stats)
+		return r
+	}
+	return nil
+}
+
+var mdMembers = regexp.MustCompile(`\[([U_]+)\]`)
+
+// mdHealth reads one array's block of /proc/mdstat: healthy when every member is up and it is not recovering.
+func mdHealth(mdstat, device string) (bool, string, bool) {
+	lines := strings.Split(mdstat, "\n")
+	for i, l := range lines {
+		if !strings.HasPrefix(l, device+" ") {
+			continue
+		}
+		block := l
+		for j := i + 1; j < len(lines) && strings.HasPrefix(lines[j], " "); j++ {
+			block += " " + strings.TrimSpace(lines[j])
+		}
+		members := mdMembers.FindStringSubmatch(block)
+		healthy := members != nil && !strings.Contains(members[1], "_") && !strings.Contains(block, "recovery")
+		return healthy, oneLineOf(block), true
+	}
+	return false, "", false
+}
+
+func firstLineOf(s string) string {
+	line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
+	return line
+}
+
+func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") }

modules/disk-load/cmd/disk-load/readings.go

@@ -0,0 +1,12 @@
+package main
+
+// The readings layout, their key and the peak are mesh-sdk's (health.Reading, health.NextReading; novox/hq ADR 0571
+// clause 3): disk-load measures each placed data item of its node and keeps the readings as its own state.
+
+import "git.novox.be/novox/mesh-sdk/go/health"
+
+// StateReadings is the state disk-load keeps.
+const StateReadings = health.ReadingsState
+
+// ClassCache is the class of an item that is listed and never measured.
+const ClassCache = "cache"

modules/disk-load/cmd/disk-load/readings_test.go

@@ -0,0 +1,144 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	"git.novox.be/novox/mesh-sdk/go/health"
+)
+
+type memStore struct{ m map[string]json.RawMessage }
+
+func (s *memStore) Get(k string) (json.RawMessage, error) { return s.m[k], nil }
+func (s *memStore) Put(k string, v any) error {
+	raw, _ := json.Marshal(v)
+	s.m[k] = raw
+	return nil
+}
+func (s *memStore) Keys() ([]string, error) {
+	var out []string
+	for k := range s.m {
+		out = append(out, k)
+	}
+	return out, nil
+}
+
+// A round measures what the data file declares but caches, keeps the readings, writes nothing the second time, and a
+// walk is not repeated within a day unless asked.
+func TestARoundMeasuresTheDeclaredItemsAndWritesWhatChanged(t *testing.T) {
+	text := "# pg\nitem data valuable /srv/pg /srv/pg backup walk\nitem tmp cache /srv/tmp - none\n# bad-line-module\nitem logs rebuildable /srv/logs - none shallow\n"
+	placed, err := catalogue.ParseDataItems(text)
+	if err != nil {
+		t.Fatal(err)
+	}
+	walks := 0
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		switch {
+		case name == "test":
+			return "", nil
+		case name == "findmnt":
+			return "", os.ErrNotExist
+		case name == "bash" && strings.Contains(args[1], "-maxdepth 1"):
+			return "1760000000 2\n", nil
+		case name == "bash":
+			walks++
+			return "4096 1760000000 3\n0\n", nil
+		}
+		return "", os.ErrNotExist
+	}
+	now := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)
+	store := &memStore{m: map[string]json.RawMessage{}}
+	k := &Keeper{Node: "ace", Items: func(context.Context) (map[string][]catalogue.PlacedDataItem, error) { return placed, nil }, Store: store, Say: func(string, ...any) {},
+		Measurer: Measurer{Run: run, Now: func() time.Time { return now }}}
+	w, kept, err := k.Round(context.Background(), false)
+	if err != nil || w != 2 || kept != 0 || len(store.m) != 2 {
+		t.Fatalf("first round: %d written %d kept %v, keys %d", w, kept, err, len(store.m))
+	}
+	now = now.Add(time.Hour)
+	if w, kept, _ := k.Round(context.Background(), false); w != 0 || kept != 2 || walks != 1 {
+		t.Fatalf("second round: %d written, %d kept, %d walks; wanted none written and the walk not repeated", w, kept, walks)
+	}
+	if _, _, err := k.MeasureNow(context.Background(), "pg", "data"); err != nil || walks != 2 {
+		t.Fatalf("measure now: %v, %d walks", err, walks)
+	}
+	// An item no longer declared is kept, retired with when and why; declared again, it is not retired any more.
+	delete(placed, "pg")
+	if _, _, err := k.Round(context.Background(), false); err != nil {
+		t.Fatal(err)
+	}
+	gone, _ := health.DecodeReading(store.m["ace.pg.data"])
+	if gone.RetiredAt.IsZero() || !strings.Contains(gone.RetiredWhy, "no longer declared on ace") || gone.FirstSeen.IsZero() {
+		t.Fatalf("a reading of an item no longer declared: %+v", gone)
+	}
+	placed["pg"] = []catalogue.PlacedDataItem{{Module: "pg", ID: "data", Class: "valuable", Path: "/srv/pg", Measure: catalogue.MeasureWalk}}
+	if _, _, err := k.Round(context.Background(), false); err != nil {
+		t.Fatal(err)
+	}
+	if back, _ := health.DecodeReading(store.m["ace.pg.data"]); !back.RetiredAt.IsZero() {
+		t.Fatalf("a reading declared again is still retired: %+v", back)
+	}
+	if _, _, err := k.MeasureNow(context.Background(), "pg", "tmp"); err == nil {
+		t.Fatal("a cache was measured")
+	}
+}
+
+// The redundant storage under an item is read: a ZFS pool healthy, then degraded; an md array with a member missing; and
+// plain storage, which is no redundancy at all. A dataset is measured from the filesystem's counters and a shallow item
+// from its top-level entries only; a walk that meets more than its bound stops and says so.
+func TestTheStorageUnderAnItemAndTheLargeMethodsAreRead(t *testing.T) {
+	poolHealth, statusX, fs := "ONLINE\n", "pool 'storage' is healthy\n", "storage/media zfs\n"
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		line := name + " " + strings.Join(args, " ")
+		switch {
+		case name == "test":
+			return "", nil
+		case name == "findmnt":
+			return fs, nil
+		case name == "zpool" && args[0] == "list":
+			return poolHealth, nil
+		case name == "zpool":
+			return statusX, nil
+		case name == "cat":
+			return "Personalities : [raid1]\nmd127 : active raid1 sdb1[1] sda1[0]\n      976630464 blocks super 1.2 [2/1] [U_]\n\nunused devices: <none>\n", nil
+		case name == "zfs" && args[len(args)-1] == "/storage/media/movies":
+			return "storage/media\t97067690722560\n", nil
+		case name == "zfs":
+			return "", os.ErrNotExist
+		case name == "bash" && strings.Contains(line, "-maxdepth 1"):
+			return "1759744800 12673\n", nil
+		case name == "bash":
+			return "123 1759700000 2000000\n141\n", nil
+		}
+		return "", nil
+	}
+	m := Measurer{Run: run, Now: time.Now}
+	ctx := context.Background()
+	got := m.Measure(ctx, catalogue.PlacedDataItem{Module: "media", ID: "movies", Path: "/storage/media/movies", Measure: catalogue.MeasureDataset})
+	if got.SizeBytes == nil || *got.SizeBytes != 97067690722560 || !strings.HasPrefix(got.Precision, "dataset storage/media") ||
+		got.LastWrite == nil || got.Redundancy == nil || got.Redundancy.Kind != "zfs" || got.Redundancy.Healthy == nil || !*got.Redundancy.Healthy {
+		t.Fatalf("the library from its dataset: %+v", got)
+	}
+	poolHealth, statusX = "DEGRADED\n", "  pool: storage\n state: DEGRADED\nstatus: One or more devices has been removed\n"
+	if r := m.redundancyOf(ctx, "/storage/media/movies"); r.Healthy == nil || *r.Healthy || !strings.Contains(r.Said, "DEGRADED") {
+		t.Fatalf("a degraded pool: %+v", r)
+	}
+	fs = "/dev/md127 ext4\n"
+	if r := m.redundancyOf(ctx, "/srv/x"); r == nil || r.Kind != "md" || r.Healthy == nil || *r.Healthy {
+		t.Fatalf("an array missing a member: %+v", r)
+	}
+	fs = "/dev/nvme0n1p3 ext4\n"
+	if r := m.redundancyOf(ctx, "/var/lib/x"); r != nil {
+		t.Fatalf("plain storage read as redundant: %+v", r)
+	}
+	if s := m.Measure(ctx, catalogue.PlacedDataItem{Path: "/mnt/plex/config", Measure: catalogue.MeasureShallow}); s.SizeBytes != nil || !strings.HasPrefix(s.Precision, "no size") {
+		t.Fatalf("a shallow item: %+v", s)
+	}
+	if w := m.Measure(ctx, catalogue.PlacedDataItem{Path: "/srv/big", Measure: catalogue.MeasureWalk}); !strings.HasPrefix(w.Precision, "partial") {
+		t.Fatalf("a walk stopped at its bound: %+v", w)
+	}
+}

modules/disk-load/cmd/disk-load/readings_tools.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+
+	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/health"
+)
+
+// ReadingTools are the tools over the state `readings`: every key, one key, how full the state is, and one item
+// measured now (a module's tools are its command line).
+func ReadingTools(k *Keeper) []stdio.Tool {
+	return []stdio.Tool{
+		{
+			Name: "disk_readings",
+			Description: "The data readings disk-load keeps for this machine's data items (ADR 0233, ADR 0571 clause 3): every key " +
+				"`<node>.<module>.<item>` with its size, newest write, measured-at, precision and seven-day peak, or one key with " +
+				"all it holds when `key` is given; and how full the state is: keys kept, their bytes, and the cap the bus puts on " +
+				"a state (none declared). Filter by `module`. (r)",
+			Input: map[string]any{
+				"key":    map[string]any{"type": "string", "description": "one key, <node>.<module>.<item>, with everything it holds"},
+				"module": map[string]any{"type": "string", "description": "only this module's items"},
+			},
+			Run: func(args map[string]any) (any, error) {
+				if key := text(args, "key"); key != "" {
+					raw, err := k.Store.Get(key)
+					if err != nil {
+						return nil, err
+					}
+					if len(raw) == 0 {
+						return nil, fmt.Errorf("no reading is kept at %q", key)
+					}
+					r, err := health.DecodeReading(raw)
+					return r, err
+				}
+				keys, err := k.Store.Keys()
+				if err != nil {
+					return nil, err
+				}
+				sort.Strings(keys)
+				module := text(args, "module")
+				type line struct {
+					Key       string         `json:"key"`
+					SizeBytes *int64         `json:"size_bytes,omitempty"`
+					PeakBytes *int64         `json:"peak_bytes,omitempty"`
+					Precision string         `json:"precision,omitempty"`
+					Error     string         `json:"error,omitempty"`
+					Reading   health.Reading `json:"-"`
+				}
+				var lines []line
+				total := 0
+				for _, key := range keys {
+					raw, err := k.Store.Get(key)
+					if err != nil {
+						return nil, err
+					}
+					total += len(raw)
+					r, err := health.DecodeReading(raw)
+					if err != nil || (module != "" && r.Module != module) {
+						continue
+					}
+					lines = append(lines, line{Key: key, SizeBytes: r.SizeBytes, PeakBytes: r.Peak, Precision: r.Precision, Error: r.Error})
+				}
+				return map[string]any{"readings": lines, "fill": map[string]any{"keys": len(keys), "bytes": total,
+					"cap": "none declared: the bus's own limit on a state applies"}}, nil
+			},
+		},
+		{
+			Name: "disk_measure",
+			Description: "Measure one data item of this machine now, whatever its method (a walk too), and write its reading when it " +
+				"changed. Reads the disks as root: a walk is bounded by time and files and says when it was partial.",
+			Input: map[string]any{
+				"module": map[string]any{"type": "string", "description": "the module that declares the item"},
+				"item":   map[string]any{"type": "string", "description": "the item's id"},
+			},
+			Run: func(args map[string]any) (any, error) {
+				module, item := text(args, "module"), text(args, "item")
+				if module == "" || item == "" {
+					return nil, fmt.Errorf("module and item are required")
+				}
+				r, written, err := k.MeasureNow(context.Background(), module, item)
+				if err != nil {
+					return nil, err
+				}
+				return map[string]any{"written": written, "reading": r, "key": health.ReadingKey(k.Node, module, item),
+					"note": strings.TrimSpace(map[bool]string{true: "the reading changed and was written", false: "the reading had not changed"}[written])}, nil
+			},
+		},
+	}
+}

modules/disk-load/cmd/disk-load/tools_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"encoding/json"
 	"os"
+	"regexp"
 	"sort"
 	"strings"
 	"testing"
@@ -35,9 +36,11 @@ func TestTheManifestNamesExactlyTheToolsTheBundleServesAndNoMachine(t *testing.T
 		t.Fatal(err)
 	}
 	var served []string
-	for _, tl := range Tools(newFake(t).machine()) {
+	k := &Keeper{Store: &memStore{m: map[string]json.RawMessage{}}}
+	for _, tl := range append(Tools(newFake(t).machine()), ReadingTools(k)...) {
 		served = append(served, tl.Name)
-		if !strings.HasSuffix(tl.Description, "(r)") {
+		// disk_measure writes the module's own state of readings and nothing else of the machine.
+		if !strings.HasSuffix(tl.Description, "(r)") && tl.Name != "disk_measure" {
 			t.Errorf("%s only reads and does not say so", tl.Name)
 		}
 	}
@@ -50,7 +53,7 @@ func TestTheManifestNamesExactlyTheToolsTheBundleServesAndNoMachine(t *testing.T
 		t.Errorf("the module only reads, and declares %d resources", len(m.Resources))
 	}
 	for _, banned := range []string{"g14", "novox", "ace", "shanks", "jochen", "/home/"} {
-		if strings.Contains(string(raw), banned) {
+		if regexp.MustCompile(`(^|[^a-z0-9-])` + regexp.QuoteMeta(banned) + `($|[^a-z0-9-])`).Match(raw) {
 			t.Errorf("the manifest says %q", banned)
 		}
 	}