Add disk-load's watched items, their test and its whole test, which the binary's ignore line hid

The ignore line `disk-load` matched the directory cmd/disk-load, so new files there were never
added; it now names the built binary alone.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL

modules/disk-load/cmd/disk-load/data.go

@@ -0,0 +1,119 @@
+package main
+
+// Where disk-load learns the data items it measures (novox/hq ADR 0577 clause 2): the node-engine's state data
+// (delivery.EngineDataBucket), one key `<node>.<module>` per module placed on this node that declares data items, as the
+// engine composed them. disk-load watches its own node's keys and composes nothing: it reads no resolved values, no
+// manifests and no placement, so a manifest its mesh-sdk cannot judge never silences it, and a round reads one watched
+// map instead of several hundred keys over the tunnel (issue 1955).
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+
+	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// Watched is this node's data items as the engine's state data holds them, by module, kept from a watch of its keys.
+type Watched struct {
+	Node string
+
+	mu       sync.Mutex
+	byKey    map[string]string // key to its module, for a delete that carries no value
+	byModule map[string][]catalogue.PlacedDataItem
+	ready    bool
+}
+
+// ErrNotWatchedYet is a round asked before the watch handed over the engine's current keys: no item is known yet, and
+// none is taken for gone.
+var ErrNotWatchedYet = errors.New("the node-engine's state " + delivery.EngineDataBucket + " is not watched yet: " +
+	"no data item of this machine is known")
+
+// Take keeps one change of a key and answers the module it is of and whether its items changed.
+func (w *Watched) Take(ch stdio.StateChange) (string, bool, error) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if w.byKey == nil {
+		w.byKey, w.byModule = map[string]string{}, map[string][]catalogue.PlacedDataItem{}
+	}
+	if ch.Op != stdio.OpPut {
+		module, known := w.byKey[ch.Key]
+		if !known {
+			return "", false, nil
+		}
+		delete(w.byKey, ch.Key)
+		delete(w.byModule, module)
+		return module, true, nil
+	}
+	d, err := delivery.DecodeEngineData(ch.Value)
+	if err != nil {
+		return "", false, fmt.Errorf("%s: %w", ch.Key, err)
+	}
+	if d.Node != w.Node {
+		return "", false, fmt.Errorf("%s names the machine %q, and this is %s", ch.Key, d.Node, w.Node)
+	}
+	for _, it := range d.Items {
+		if it.Module != d.Module {
+			return "", false, fmt.Errorf("%s holds an item of %q, and is %s's", ch.Key, it.Module, d.Module)
+		}
+	}
+	changed := !sameItems(w.byModule[d.Module], d.Items)
+	w.byKey[ch.Key] = d.Module
+	w.byModule[d.Module] = d.Items
+	return d.Module, changed, nil
+}
+
+// Ready says the watch handed over every current key: from now on an item missing is an item gone.
+func (w *Watched) Ready() {
+	w.mu.Lock()
+	w.ready = true
+	w.mu.Unlock()
+}
+
+// All is every module's items, as the Keeper reads them; refused until the watch is ready.
+func (w *Watched) All(context.Context) (map[string][]catalogue.PlacedDataItem, error) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if !w.ready {
+		return nil, ErrNotWatchedYet
+	}
+	out := make(map[string][]catalogue.PlacedDataItem, len(w.byModule))
+	for m, its := range w.byModule {
+		out[m] = append([]catalogue.PlacedDataItem(nil), its...)
+	}
+	return out, nil
+}
+
+func sameItems(a, b []catalogue.PlacedDataItem) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// Follow watches this node's keys of the engine's state data, and asks the keeper to measure a module whose items changed
+// after the watch began; it returns once the current keys were handed over (stdio.KeptState.Follow).
+func (w *Watched) Follow(k *Keeper) error {
+	pattern, err := delivery.EngineDataNodeKeys(w.Node)
+	if err != nil {
+		return err
+	}
+	stdio.State(delivery.EngineDataRead).Follow(pattern, func(ch stdio.StateChange) error {
+		module, changed, err := w.Take(ch)
+		if err != nil || !changed || ch.Current {
+			return err
+		}
+		k.Due(module)
+		return nil
+	})
+	w.Ready()
+	return nil
+}

modules/disk-load/cmd/disk-load/data_test.go

@@ -0,0 +1,153 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"git.novox.be/novox/mesh-sdk/go/health"
+)
+
+// ADR 0577 clauses 2, 3 (and the disk-load check of clause 5): the items measured come from the node-engine's keys of
+// this machine as the watch hands them over, and none is known (none taken for gone) before the watch caught up; a key
+// that changes after the watch began has its module measured again, and one deleted is measured no more; an item whose
+// measurement fails is a failed item of the round, which measures the rest, and the round's key says so.
+func TestTheItemsComeFromTheEnginesKeysAndARoundSaysHowFarItGot(t *testing.T) {
+	key := func(module string) string { k, _ := delivery.EngineDataKey("ace", module); return k }
+	value := func(module string, items ...catalogue.PlacedDataItem) json.RawMessage {
+		raw, err := delivery.EncodeEngineData(delivery.EngineData{Node: "ace", Module: module, Digest: "sha256:" + module, Items: items})
+		if err != nil {
+			t.Fatal(err)
+		}
+		return raw
+	}
+	pg := catalogue.PlacedDataItem{Module: "pg", ID: "data", Class: "valuable", Path: "/srv/pg", Measure: catalogue.MeasureShallow}
+	gone := catalogue.PlacedDataItem{Module: "pg", ID: "wal", Class: "valuable", Path: "/srv/missing", Measure: catalogue.MeasureShallow}
+	web := catalogue.PlacedDataItem{Module: "web", ID: "uploads", Class: "valuable", Path: "/srv/web", Measure: catalogue.MeasureShallow}
+	measured := map[string]int{}
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		switch {
+		case name == "test" && args[1] == "/srv/missing":
+			return "", errors.New("no such file")
+		case name == "test":
+			return "", nil
+		case name == "bash" && strings.Contains(args[1], "-maxdepth 1"):
+			measured[strings.Split(args[1], "'")[1]]++
+			return "1760000000 2\n", nil
+		}
+		return "", os.ErrNotExist
+	}
+	now := time.Date(2026, 11, 1, 0, 30, 0, 0, time.UTC)
+	readings, rounds := &memStore{m: map[string]json.RawMessage{}}, &memStore{m: map[string]json.RawMessage{}}
+	w := &Watched{Node: "ace"}
+	var said []string
+	k := &Keeper{Node: "ace", Items: w.All, Store: readings, Rounds: rounds,
+		Say:      func(f string, a ...any) { said = append(said, f) },
+		Measurer: Measurer{Run: run, Now: func() time.Time { return now }}}
+	round := func() Round {
+		var r Round
+		raw, _ := rounds.Get("ace")
+		if err := json.Unmarshal(raw, &r); err != nil {
+			t.Fatalf("the round's key: %v", err)
+		}
+		return r
+	}
+
+	// Before the watch caught up nothing is known, and a round says so in its key rather than retiring every reading.
+	if _, _, err := k.Round(context.Background(), false); !errors.Is(err, ErrNotWatchedYet) {
+		t.Fatalf("a round before the watch: %v", err)
+	}
+	if r := round(); r.Ended == nil || !strings.Contains(r.Error, "not watched yet") {
+		t.Fatalf("the round before the watch is kept as %+v", r)
+	}
+	for _, step := range []struct {
+		name    string
+		change  stdio.StateChange
+		module  string
+		changed bool
+		err     bool
+	}{
+		{"a current key", stdio.StateChange{Key: key("pg"), Op: stdio.OpPut, Value: value("pg", pg, gone), Current: true}, "pg", true, false},
+		{"another current key", stdio.StateChange{Key: key("web"), Op: stdio.OpPut, Value: value("web", web), Current: true}, "web", true, false},
+		{"the same items again", stdio.StateChange{Key: key("web"), Op: stdio.OpPut, Value: value("web", web)}, "web", false, false},
+		{"another machine's items", stdio.StateChange{Key: key("x"), Op: stdio.OpPut, Value: json.RawMessage(
+			`{"v":1,"node":"g14","module":"x","digest":"d","items":[]}`)}, "", false, true},
+		{"a delete of a key never seen", stdio.StateChange{Key: key("y"), Op: stdio.OpDelete}, "", false, false},
+	} {
+		module, changed, err := w.Take(step.change)
+		if module != step.module || changed != step.changed || (err != nil) != step.err {
+			t.Errorf("%s: taken as %q, changed %v, %v", step.name, module, changed, err)
+		}
+	}
+	w.Ready()
+
+	// A round measures every item it was handed; the missing one fails, and the round goes on and says it.
+	if _, _, err := k.Round(context.Background(), false); err != nil {
+		t.Fatalf("the round: %v", err)
+	}
+	r := round()
+	if r.Ended == nil || r.Items != 3 || r.Measured != 3 || len(r.Failed) != 1 || r.Failed[0].Item != "pg.wal" ||
+		!strings.Contains(r.Failed[0].Why, "does not exist") || measured["/srv/pg"] != 1 || measured["/srv/web"] != 1 {
+		t.Fatalf("the round's key says %+v; measured %v", r, measured)
+	}
+	if !strings.Contains(strings.Join(said, "\n"), "the round ended") || !strings.Contains(strings.Join(said, "\n"), "starts") {
+		t.Errorf("the journal said %v", said)
+	}
+
+	// A key that changes after the watch began has its module measured again, at once, by Work.
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go k.Work(ctx)
+	moved := web
+	moved.Path = "/srv/web2"
+	if module, changed, err := w.Take(stdio.StateChange{Key: key("web"), Op: stdio.OpPut, Value: value("web", moved)}); err != nil || !changed {
+		t.Fatalf("a changed key: %s %v %v", module, changed, err)
+	}
+	k.Due("web")
+	waitFor(t, func() bool {
+		raw := readingOf(readings, "ace.web.uploads")
+		return raw != nil && raw.Path == "/srv/web2"
+	})
+	// A key deleted: its module's items are measured no more, and its reading is retired.
+	if module, changed, _ := w.Take(stdio.StateChange{Key: key("web"), Op: stdio.OpDelete}); module != "web" || !changed {
+		t.Fatalf("a deleted key was taken as %q, %v", module, changed)
+	}
+	k.Due("web")
+	waitFor(t, func() bool { r := readingOf(readings, "ace.web.uploads"); return r != nil && !r.RetiredAt.IsZero() })
+	before := measured["/srv/web2"]
+	if _, _, err := k.Round(context.Background(), false); err != nil {
+		t.Fatal(err)
+	}
+	if measured["/srv/web2"] != before || round().Items != 2 {
+		t.Errorf("an item whose key was deleted was measured again: %d times, round %+v", measured["/srv/web2"]-before, round())
+	}
+}
+
+func readingOf(s *memStore, key string) *health.Reading {
+	raw, _ := s.Get(key)
+	if raw == nil {
+		return nil
+	}
+	r, err := health.DecodeReading(raw)
+	if err != nil {
+		return nil
+	}
+	return &r
+}
+
+func waitFor(t *testing.T, done func() bool) {
+	t.Helper()
+	for end := time.Now().Add(5 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) {
+		if done() {
+			return
+		}
+	}
+	t.Fatal("not done within 5 s")
+}

modules/disk-load/cmd/disk-load/whole_integration_test.go

@@ -0,0 +1,267 @@
+//go:build integration
+
+package main
+
+// disk-load whole against a real bus (novox/hq ADR 0577 clause 5, the group's integration test): the built binary, played
+// over its real stdin and stdout as the node's tool runtime plays it (mesh/state.* answered from a real nats-server's
+// key-value buckets, a watch handed over from the bus's own watch, mesh/publish onto its subjects). A key of the
+// node-engine's state data written after disk-load started has its module's item read within a minute, and disk-load
+// composes nothing to learn it.
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"git.novox.be/novox/mesh-sdk/go/health"
+	"git.novox.be/novox/mesh-sdk/go/testbus"
+)
+
+type wire struct {
+	ID     json.RawMessage `json:"id,omitempty"`
+	Method string          `json:"method,omitempty"`
+	Params json.RawMessage `json:"params,omitempty"`
+	Result json.RawMessage `json:"result,omitempty"`
+}
+
+// player is the node's tool runtime, as far as disk-load needs it, over a real bus.
+type player struct {
+	ctx   context.Context
+	nc    *nats.Conn
+	js    jetstream.JetStream
+	mu    sync.Mutex
+	in    io.Writer
+	next  int
+	asked []string // every state call, `<method> <state> <key>`
+}
+
+func (p *player) send(m map[string]any) {
+	raw, _ := json.Marshal(m)
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	_, _ = p.in.Write(append(raw, '\n'))
+}
+
+func (p *player) answer(id json.RawMessage, result any, err error) {
+	if err != nil {
+		p.send(map[string]any{"jsonrpc": "2.0", "id": id, "error": map[string]any{"code": -32000, "message": err.Error()}})
+		return
+	}
+	p.send(map[string]any{"jsonrpc": "2.0", "id": id, "result": result})
+}
+
+// bucket is a state's bucket: another module's `<owner>.<name>`, or disk-load's own by its local name.
+func bucket(state string) string {
+	if owner, name, other := strings.Cut(state, "."); other {
+		return owner + "_" + name
+	}
+	return "disk-load_" + state
+}
+
+func (p *player) serve(out io.Reader) {
+	scanner := bufio.NewScanner(out)
+	scanner.Buffer(make([]byte, 1<<20), 1<<24)
+	for scanner.Scan() {
+		var m wire
+		if json.Unmarshal(scanner.Bytes(), &m) != nil || m.Method == "" {
+			continue // an answer to a change handed over
+		}
+		go p.handle(m)
+	}
+}
+
+func (p *player) handle(m wire) {
+	var a struct {
+		State string          `json:"state"`
+		Key   string          `json:"key"`
+		Value json.RawMessage `json:"value"`
+		Body  json.RawMessage `json:"body"`
+	}
+	_ = json.Unmarshal(m.Params, &a)
+	p.mu.Lock()
+	p.asked = append(p.asked, m.Method+" "+a.State+" "+a.Key)
+	p.mu.Unlock()
+	if m.Method == "mesh/publish" {
+		var e struct {
+			Key  string          `json:"key"`
+			Body json.RawMessage `json:"body"`
+		}
+		_ = json.Unmarshal(m.Params, &e)
+		p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod.disk-load.event."+e.Key, e.Body))
+		return
+	}
+	if !strings.HasPrefix(m.Method, "mesh/state.") {
+		p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
+		return
+	}
+	kv, err := p.js.KeyValue(p.ctx, bucket(a.State))
+	if err != nil {
+		p.answer(m.ID, nil, err)
+		return
+	}
+	switch m.Method {
+	case "mesh/state.get":
+		e, err := kv.Get(p.ctx, a.Key)
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			p.answer(m.ID, nil, nil)
+			return
+		}
+		if err != nil {
+			p.answer(m.ID, nil, err)
+			return
+		}
+		p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil)
+	case "mesh/state.put":
+		rev, err := kv.Put(p.ctx, a.Key, a.Value)
+		p.answer(m.ID, map[string]any{"revision": rev}, err)
+	case "mesh/state.keys":
+		keys, err := kv.Keys(p.ctx)
+		if errors.Is(err, jetstream.ErrNoKeysFound) {
+			keys, err = []string{}, nil
+		}
+		p.answer(m.ID, keys, err)
+	case "mesh/state.watch":
+		p.watch(m.ID, a.State, kv, a.Key)
+	default:
+		p.answer(m.ID, nil, errors.New("not played here"))
+	}
+}
+
+// watch hands over the bus's watch of the pattern: every current value, then the answer to the watch, then each change.
+func (p *player) watch(id json.RawMessage, state string, kv jetstream.KeyValue, pattern string) {
+	var watchID string
+	_ = json.Unmarshal(id, &watchID)
+	if pattern == "" {
+		pattern = ">"
+	}
+	w, err := kv.Watch(p.ctx, strings.ReplaceAll(pattern, "**", ">"))
+	if err != nil {
+		p.answer(id, nil, err)
+		return
+	}
+	current := true
+	for e := range w.Updates() {
+		if e == nil {
+			current = false
+			p.answer(id, map[string]any{}, nil)
+			continue
+		}
+		op := "put"
+		if e.Operation() != jetstream.KeyValuePut {
+			op = "delete"
+		}
+		p.mu.Lock()
+		p.next++
+		n := p.next
+		p.mu.Unlock()
+		p.send(map[string]any{"jsonrpc": "2.0", "id": fmt.Sprintf("rt-%d", n), "method": "mesh/state", "params": map[string]any{
+			"watch": watchID, "change": map[string]any{"state": state, "key": e.Key(), "op": op,
+				"value": json.RawMessage(e.Value()), "revision": e.Revision(), "current": current}}})
+	}
+}
+
+// 0577/AC5 (the group's integration test): a key of the node-engine's state data written after disk-load started is
+// read within a minute, its item's reading is on the bus, and disk-load read no composition input to learn it.
+func TestDiskLoadWholeMeasuresAnItemWithinAMinuteOfItsKey(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+	bin := filepath.Join(t.TempDir(), "disk-load")
+	build := exec.Command("go", "build", "-o", bin, ".")
+	build.Stderr = os.Stderr
+	if err := build.Run(); err != nil {
+		t.Fatalf("disk-load does not build: %v", err)
+	}
+	nc, err := nats.Connect(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(nc.Close)
+	js, err := jetstream.New(nc)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, b := range []string{delivery.EngineDataBucket, health.ReadingsBucket, "disk-load_" + StateRounds} {
+		if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	cmd := exec.CommandContext(ctx, bin)
+	cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE=disk-load")
+	stdin, _ := cmd.StdinPipe()
+	stdout, _ := cmd.StdoutPipe()
+	cmd.Stderr = os.Stderr
+	if err := cmd.Start(); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() })
+	p := &player{ctx: ctx, nc: nc, js: js, in: stdin}
+	go p.serve(stdout)
+
+	// Once disk-load watches its machine's keys, the engine writes one.
+	watching := func() bool {
+		p.mu.Lock()
+		defer p.mu.Unlock()
+		for _, a := range p.asked {
+			if a == "mesh/state.watch "+delivery.EngineDataRead+" ace.*" {
+				return true
+			}
+		}
+		return false
+	}
+	for end := time.Now().Add(20 * time.Second); !watching(); time.Sleep(50 * time.Millisecond) {
+		if time.Now().After(end) {
+			t.Fatalf("disk-load did not watch its machine's keys of %s: asked %v", delivery.EngineDataBucket, p.asked)
+		}
+	}
+	data, _ := js.KeyValue(ctx, delivery.EngineDataBucket)
+	dir := t.TempDir()
+	item := catalogue.PlacedDataItem{Module: "pg", ID: "data", Class: "valuable", Path: dir, Measure: catalogue.MeasureShallow}
+	raw, err := delivery.EncodeEngineData(delivery.EngineData{Node: "ace", Module: "pg", Digest: "sha256:pg", Items: []catalogue.PlacedDataItem{item}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	key, _ := delivery.EngineDataKey("ace", "pg")
+	written := time.Now()
+	if _, err := data.Put(ctx, key, raw); err != nil {
+		t.Fatal(err)
+	}
+	readings, _ := js.KeyValue(ctx, health.ReadingsBucket)
+	for {
+		e, err := readings.Get(ctx, health.ReadingKey("ace", "pg", "data"))
+		if err == nil {
+			r, err := health.DecodeReading(e.Value())
+			if err != nil || r.Path != dir || r.MeasuredAt == nil {
+				t.Fatalf("the reading is %s (%v)", e.Value(), err)
+			}
+			break
+		}
+		if time.Since(written) > time.Minute {
+			t.Fatalf("no reading within a minute of the key's write: %v", err)
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	// It composed nothing: every state it asked is the engine's data or its own.
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	for _, a := range p.asked {
+		f := strings.Fields(a)
+		if len(f) > 1 && strings.HasPrefix(f[0], "mesh/state.") && f[1] != delivery.EngineDataRead && f[1] != StateReadings && f[1] != StateRounds {
+			t.Errorf("disk-load read %s, which is no state of its own nor the engine's data", a)
+		}
+	}
+}