Design 47's last milestones: the keeper, self-check, healers, hand-acts, provisioner, vault, artifact sweep, disk readings, credential forms (issues 774, 1952, 1953, 1954, 1955)

One commit with exactly the tree of the reviewed head 35b4509c0a, on main 12c2574c9d, so two compiled programs added and removed on the branch leave no trace in history (issue 1577).

modules/artifact-store-tools/cmd/artifact-store-tools/door.go

@@ -0,0 +1,129 @@
+package main
+
+// The artifact store's door and the bus, as the holder reaches them through this module's runtime.
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	stdio "git.novox.be/novox/mesh-sdk/go"
+
+	"artifactstoretools/internal/registry"
+)
+
+// door is registry.Store at the store's address as this machine reaches it.
+type door struct{ s registry.Store }
+
+func (d *door) Address() string                                  { return d.s.Address }
+func (d *door) Hold(ctx context.Context, r string) (bool, error) { return d.s.Hold(ctx, r) }
+func (d *door) Holds(ctx context.Context, r string) error        { return d.s.Holds(ctx, r) }
+func (d *door) IsIndex(ctx context.Context, r string) (bool, error) {
+	return d.s.IsIndex(ctx, r)
+}
+func (d *door) LetGo(ctx context.Context, r string) error { return d.s.LetGo(ctx, r) }
+func (d *door) SpareKept(ctx context.Context, keptImages, touching []string) error {
+	spare, err := d.s.SpareKeptIndexes(ctx, keptImages, touching)
+	if err == nil {
+		d.s.Spare = spare
+	}
+	return err
+}
+
+// DoorAt reads the store's address (host:port) from the file the manifest fills from the seat mesh-artifact-store's
+// reach, each time, so a store that moved is reached where it is now.
+func DoorAt(file string) func() (Door, error) {
+	return func() (Door, error) {
+		if file == "" {
+			return nil, fmt.Errorf("this bundle was given no file naming the artifact store's address")
+		}
+		raw, err := os.ReadFile(file)
+		if err != nil {
+			return nil, fmt.Errorf("the artifact store's address: %w", err)
+		}
+		address := strings.TrimSpace(string(raw))
+		if address == "" {
+			return nil, fmt.Errorf("the artifact store's address is empty in %s", file)
+		}
+		return &door{s: registry.Store{Address: address}}, nil
+	}
+}
+
+// runtimeBus is the holder's bus through the runtime: its reads, its findings and its seat's event.
+type runtimeBus struct{}
+
+func (runtimeBus) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
+func (runtimeBus) Get(state, key string) ([]byte, bool, error) {
+	e, err := stdio.State(state).Get(key)
+	if err != nil || e == nil {
+		return nil, false, err
+	}
+	return e.Value, true, nil
+}
+func (runtimeBus) History(state, key string) ([]stdio.StateRevision, error) {
+	return stdio.State(state).History(key)
+}
+func (runtimeBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) {
+	var out []stdio.LogEntry
+	page, err := stdio.Log(read).Since(at, "", 0)
+	for err == nil {
+		out = append(out, page.Entries...)
+		if page.Next == 0 {
+			return out, nil
+		}
+		page, err = stdio.Log(read).Read(page.Next, "", 0)
+	}
+	return nil, err
+}
+
+func (runtimeBus) Put(state, key string, value any) error {
+	_, err := stdio.State(state).Put(key, value)
+	return err
+}
+func (runtimeBus) Delete(state, key string) error { return stdio.State(state).Delete(key) }
+func (runtimeBus) Publish(subject string, body any) error {
+	return stdio.SeatPublish(subject, body, "")
+}
+
+// follow watches the builder's done keys and the wants: a build's end sweeps (once the lease names no controller) and
+// observes again; a want changed observes again. Event-driven: nothing reads them on a timer.
+func follow(h *Holder) {
+	<-stdio.Served()
+	watch := func(state string, handle func(stdio.StateChange) error) {
+		for {
+			if err := stdio.State(state).Watch("", handle); err == nil {
+				return
+			} else {
+				h.logf("watching %s: %v; asked again in 10s", state, err)
+			}
+			time.Sleep(10 * time.Second)
+		}
+	}
+	observe := func() {
+		if err := h.Observe(context.Background()); err != nil {
+			h.logf("image-missing could not be observed: %v", err)
+		}
+	}
+	watch(BuildsRead, func(c stdio.StateChange) error {
+		if c.Current || c.Op != stdio.OpPut {
+			return nil
+		}
+		var b struct {
+			State string `json:"state"`
+		}
+		if json.Unmarshal(c.Value, &b) == nil && b.State == "done" {
+			go func() { h.AfterBuild(c.Key); observe() }()
+		}
+		return nil
+	})
+	watch(WantsRead, func(c stdio.StateChange) error {
+		if !c.Current {
+			go observe()
+		}
+		return nil
+	})
+	observe()
+}

modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go

@@ -0,0 +1,260 @@
+//go:build integration
+
+package main
+
+// The seat mesh-artifact-sweep's holder whole against a real bus (novox/hq ADR 0571 clause 2, issue 1955): the builder's done
+// keys, the wants and the controller's lease in a real nats-server's key-value buckets, a build's end heard by a watch
+// of the builds as the runtime hands it over, each deletion recorded by a request to the seat mesh-hand-acts on the bus,
+// artifacts-gone said on the seat's subject, and S25's finding written to the holder's own bucket. Only the store's
+// door is in memory: its HTTP is the copied registry package's, tested there.
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+
+	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/buscompose"
+	"git.novox.be/novox/mesh-sdk/go/handacts"
+	"git.novox.be/novox/mesh-sdk/go/testbus"
+)
+
+// realBus is the holder's Bus over a real nats-server: each read state a bucket named as the runtime names it.
+type realBus struct {
+	ctx context.Context
+	nc  *nats.Conn
+	js  jetstream.JetStream
+	// log is the builder's log as the runtime's log verbs would read it (mesh-tools adds them; the sdk's own read of a
+	// log is tested in mesh-sdk's process package).
+	log []stdio.LogEntry
+}
+
+func (b realBus) LogSince(_ string, at time.Time) ([]stdio.LogEntry, error) {
+	var out []stdio.LogEntry
+	for _, e := range b.log {
+		if !e.At.Before(at) {
+			out = append(out, e)
+		}
+	}
+	return out, nil
+}
+
+func bucketOf(state string) string { return strings.Replace(state, ".", "_", 1) }
+
+func (b realBus) kv(state string) (jetstream.KeyValue, error) {
+	return b.js.KeyValue(b.ctx, bucketOf(state))
+}
+
+func (b realBus) Keys(state string) ([]string, error) {
+	kv, err := b.kv(state)
+	if err != nil {
+		return nil, err
+	}
+	keys, err := kv.Keys(b.ctx)
+	if err == jetstream.ErrNoKeysFound {
+		return nil, nil
+	}
+	return keys, err
+}
+
+func (b realBus) Get(state, key string) ([]byte, bool, error) {
+	kv, err := b.kv(state)
+	if err != nil {
+		return nil, false, err
+	}
+	e, err := kv.Get(b.ctx, key)
+	if err == jetstream.ErrKeyNotFound {
+		return nil, false, nil
+	}
+	if err != nil {
+		return nil, false, err
+	}
+	return e.Value(), true, nil
+}
+
+func (b realBus) History(state, key string) ([]stdio.StateRevision, error) {
+	kv, err := b.kv(state)
+	if err != nil {
+		return nil, err
+	}
+	h, err := kv.History(b.ctx, key)
+	if err != nil {
+		return nil, err
+	}
+	var out []stdio.StateRevision
+	for _, e := range h {
+		out = append(out, stdio.StateRevision{Key: key, Value: e.Value(), Revision: e.Revision(), At: e.Created()})
+	}
+	return out, nil
+}
+
+func (b realBus) Put(state, key string, value any) error {
+	kv, err := b.kv(state)
+	if err != nil {
+		return err
+	}
+	raw, _ := json.Marshal(value)
+	_, err = kv.Put(b.ctx, key, raw)
+	return err
+}
+
+func (b realBus) Delete(state, key string) error {
+	kv, err := b.kv(state)
+	if err != nil {
+		return err
+	}
+	return kv.Delete(b.ctx, key)
+}
+
+func (b realBus) Publish(subject string, body any) error {
+	raw, _ := json.Marshal(body)
+	return b.nc.Publish(subject, raw)
+}
+
+func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	nc, err := nats.Connect(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer nc.Close()
+	js, err := jetstream.New(nc)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, s := range []string{BuildsRead, WantsRead, HavesRead, LeaseRead, ManifestsRead, "artifact-store-tools." + FindingsState} {
+		if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucketOf(s), History: 5}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	bus := realBus{ctx: ctx, nc: nc, js: js}
+
+	// The mesh as aMesh lays it, copied onto the real buckets: letta's seven builds, a want, the store's listing.
+	mem, memb, door, _ := aMesh(t, &buscompose.LeaseHolder{Instance: "c1", Host: "novox", Epoch: 3})
+	bus.log = memb.log
+	for _, s := range []string{BuildsRead, WantsRead, LeaseRead, ManifestsRead} {
+		keys, _ := memb.Keys(s)
+		for _, k := range keys[:min(len(keys), 6)] { // the seventh build ends later, on the bus
+			raw, _, _ := memb.Get(s, k)
+			if err := bus.Put(s, k, json.RawMessage(raw)); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	time.Sleep(20 * time.Millisecond) // each build's end is its key's time on the bus: the seventh the newest
+
+	// The seat mesh-hand-acts, answering every act it records; artifacts-gone heard on the seat's subject.
+	var mu sync.Mutex
+	var acts, gone []string
+	if _, err := nc.Subscribe("mesh.seat."+handacts.Seat+".tool.hand-act", func(m *nats.Msg) {
+		mu.Lock()
+		acts = append(acts, string(m.Data))
+		mu.Unlock()
+		_ = m.Respond([]byte(`{"ok":true}`))
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := nc.Subscribe(GoneSubject, func(m *nats.Msg) {
+		mu.Lock()
+		gone = append(gone, string(m.Data))
+		mu.Unlock()
+	}); err != nil {
+		t.Fatal(err)
+	}
+	h := &Holder{Bus: findingsAt{realBus: bus}, Door: mem.Door, Listed: mem.Listed, Now: time.Now, Logf: t.Logf}
+	h.Acts = handacts.Recorder{LeaseAway: h.LeaseAway, Ask: func(key string, body any) (json.RawMessage, error) {
+		raw, _ := json.Marshal(body)
+		m, err := nc.Request("mesh.seat."+strings.TrimPrefix(strings.Replace(key, ".", ".tool.", 1), "seat:"), raw, 5*time.Second)
+		if err != nil {
+			return nil, err
+		}
+		return m.Data, nil
+	}}
+
+	// A build's end heard from the builds bucket, as follow hands it over.
+	kv, _ := bus.kv(BuildsRead)
+	w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
+	if err != nil {
+		t.Fatal(err)
+	}
+	ended := func() {
+		keys, _ := memb.Keys(BuildsRead)
+		raw, _, _ := memb.Get(BuildsRead, keys[6])
+		if err := bus.Put(BuildsRead, keys[6], json.RawMessage(raw)); err != nil {
+			t.Fatal(err)
+		}
+		select {
+		case e := <-w.Updates():
+			h.AfterBuild(e.Key())
+		case <-ctx.Done():
+			t.Fatal("the build's end was not heard")
+		}
+	}
+
+	// While the lease names the controller, a build's end lets nothing go.
+	ended()
+	if len(door.asked) != 0 {
+		t.Fatalf("before the lease write the store was asked %v", door.asked)
+	}
+	// The lease handed away: the next build's end holds the kept archives and lets go of letta's 2nd, which the 7th moved
+	// past the five newest and no want names (the want keeps the 1st), recording each deletion.
+	if err := bus.Put(LeaseRead, buscompose.ControllerLeaseKey, buscompose.LeaseHolder{HandedTo: "mesh-grants", Why: "the lease write"}); err != nil {
+		t.Fatal(err)
+	}
+	ended()
+	deadline := time.Now().Add(5 * time.Second)
+	for {
+		mu.Lock()
+		n := len(gone)
+		mu.Unlock()
+		if n > 0 || time.Now().After(deadline) {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	if len(acts) != 3 || !strings.Contains(acts[0], archiveRef("letta", 2)) || !strings.Contains(acts[1], imageRef("letta", 2)) ||
+		!strings.Contains(acts[2], mirror(1)) {
+		t.Fatalf("the hand acts recorded are %v; the store was asked %v", acts, door.asked)
+	}
+	if len(gone) != 1 || !strings.Contains(gone[0], `"fingerprint":"`+fp(2)+`"`) {
+		t.Fatalf("artifacts-gone said %v", gone)
+	}
+
+	// S25 through the real findings bucket: letta's wanted image missing from the store is a finding there.
+	view, _ := mem.Listed(ctx)
+	repo, digest, _ := splitReference(imageRef("letta", 1))
+	delete(view.L.Repos[repo].Revisions, digest)
+	if err := h.Observe(ctx); err != nil {
+		t.Fatal(err)
+	}
+	if keys, _ := bus.Keys("artifact-store-tools." + FindingsState); len(keys) != 1 || keys[0] != "module.letta.ace.image-missing" {
+		t.Fatalf("the findings are %v", keys)
+	}
+}
+
+// findingsAt reads and writes the holder's own findings in its bucket, artifact-store-tools_findings.
+type findingsAt struct{ realBus }
+
+func (f findingsAt) own(state string) string {
+	if state == FindingsState {
+		return Module + "." + state
+	}
+	return state
+}
+func (f findingsAt) Keys(state string) ([]string, error) { return f.realBus.Keys(f.own(state)) }
+func (f findingsAt) Get(state, key string) ([]byte, bool, error) {
+	return f.realBus.Get(f.own(state), key)
+}
+func (f findingsAt) Put(state, key string, value any) error {
+	return f.realBus.Put(f.own(state), key, value)
+}
+func (f findingsAt) Delete(state, key string) error { return f.realBus.Delete(f.own(state), key) }

modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go

@@ -0,0 +1,351 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/buscompose"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"git.novox.be/novox/mesh-sdk/go/handacts"
+	"git.novox.be/novox/mesh-sdk/go/health"
+
+	"artifactstoretools/internal/registry"
+)
+
+// memBus is the holder's bus in memory: states by name, each key's history, and what was published.
+type memBus struct {
+	mu        sync.Mutex
+	states    map[string]map[string][]byte
+	history   map[string][]stdio.StateRevision
+	log       []stdio.LogEntry
+	published []string
+}
+
+func (b *memBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) {
+	var out []stdio.LogEntry
+	for _, e := range b.log {
+		if read == delivery.BuildLogRead && !e.At.Before(at) {
+			out = append(out, e)
+		}
+	}
+	return out, nil
+}
+
+func (b *memBus) Keys(state string) ([]string, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	var out []string
+	for k := range b.states[state] {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out, nil
+}
+
+func (b *memBus) Get(state, key string) ([]byte, bool, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	v, ok := b.states[state][key]
+	return v, ok, nil
+}
+
+func (b *memBus) History(state, key string) ([]stdio.StateRevision, error) {
+	return b.history[state+" "+key], nil
+}
+
+func (b *memBus) Put(state, key string, value any) error {
+	raw, _ := json.Marshal(value)
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.states[state] == nil {
+		b.states[state] = map[string][]byte{}
+	}
+	b.states[state][key] = raw
+	return nil
+}
+
+func (b *memBus) Delete(state, key string) error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	delete(b.states[state], key)
+	return nil
+}
+
+func (b *memBus) Publish(subject string, body any) error {
+	raw, _ := json.Marshal(body)
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.published = append(b.published, subject+" "+string(raw))
+	return nil
+}
+
+// memDoor is the store's door in memory: what it holds, which references are indexes, and what was asked.
+type memDoor struct {
+	holds   map[string]bool
+	indexes map[string]bool
+	asked   []string
+}
+
+func (d *memDoor) Address() string { return "10.0.0.5:5000" }
+func (d *memDoor) Hold(_ context.Context, r string) (bool, error) {
+	d.asked = append(d.asked, "hold "+r)
+	if !d.holds[r] {
+		return false, registry.Gone
+	}
+	return false, nil
+}
+func (d *memDoor) Holds(_ context.Context, r string) error {
+	if !d.holds[r] {
+		return registry.Gone
+	}
+	return nil
+}
+func (d *memDoor) IsIndex(_ context.Context, r string) (bool, error) { return d.indexes[r], nil }
+func (d *memDoor) LetGo(_ context.Context, r string) error {
+	d.asked = append(d.asked, "let go "+r)
+	delete(d.holds, r)
+	return nil
+}
+func (d *memDoor) SpareKept(context.Context, []string, []string) error {
+	d.asked = append(d.asked, "spare")
+	return nil
+}
+
+// mirror is a copy of an upstream base in the store, the n-th.
+func mirror(n int) string {
+	return "artifact-store://upstream/docker.io/library/alpine@sha256:" + strings.Repeat(fmt.Sprintf("%x", n%16), 64)
+}
+
+// pinnedImage is the image letta's manifest names by itself, at the store's address.
+const pinnedImage = "10.0.0.5:5000/letta/pinned@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
+
+func digestOf(ref string) string { _, d, _ := splitReference(ref); return d }
+
+func fp(n int) string { return "src2:" + strings.Repeat(fmt.Sprintf("%x", n%16), 64) }
+
+func imageRef(module string, n int) string {
+	return "artifact-store://" + module + "/image@sha256:" + strings.Repeat(fmt.Sprintf("%x", n%16), 64)
+}
+
+func archiveRef(module string, n int) string {
+	return "artifact-store://" + module + "/files/blobs/sha256:" + strings.Repeat(fmt.Sprintf("%x", (n+8)%16), 64)
+}
+
+// A mesh with seven done builds of letta, the n-th ending n hours after a day began, each with an image and an archive;
+// a want names the first on ace; the store lists them all and one image no build names; the lease as given.
+func aMesh(t *testing.T, lease *buscompose.LeaseHolder) (*Holder, *memBus, *memDoor, *[]string) {
+	t.Helper()
+	day := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
+	b := &memBus{states: map[string]map[string][]byte{}, history: map[string][]stdio.StateRevision{}}
+	door := &memDoor{holds: map[string]bool{}, indexes: map[string]bool{}}
+	listing := &Listing{Blobs: map[string]int64{}, Repos: map[string]*Repo{}}
+	for n := 1; n <= 7; n++ {
+		key, _ := delivery.BuilderBuildKey("letta", fp(n))
+		raw, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: strings.Repeat("d", 64), Artifacts: []delivery.BuiltArtifact{
+			{Name: "image", Kind: "image", Digest: digestOf(imageRef("letta", n)), Reference: imageRef("letta", n)},
+			{Name: "files", Kind: "archive", Digest: digestOf(archiveRef("letta", n)), Reference: archiveRef("letta", n)},
+			{Name: "pkg", Kind: "package", Digest: "sha256:" + strings.Repeat("f", 64), Reference: "letta@1.0." + fmt.Sprint(n)}}})
+		if err != nil {
+			t.Fatal(err)
+		}
+		_ = b.Put(BuildsRead, key, json.RawMessage(raw))
+		b.history[BuildsRead+" "+key] = []stdio.StateRevision{{Key: key, Value: raw, At: day.Add(time.Duration(n) * time.Hour)}}
+		for _, r := range []string{imageRef("letta", n), archiveRef("letta", n)} {
+			door.holds[r] = true
+			repo, digest, kind := splitReference(r)
+			if kind == "archive" {
+				listing.Blobs[digest] = 10
+			} else {
+				listing.repo(repo).Revisions[digest] = true
+			}
+		}
+	}
+	listing.repo("letta/image").Revisions["sha256:"+strings.Repeat("e", 64)] = true // no build names it
+	listing.repo("letta/pinned").Revisions["sha256:"+strings.Repeat("c", 64)] = true
+	// Three copies of a base: letta's 2nd stood on the first, its 7th on the second, and no build on the third.
+	for n := 1; n <= 3; n++ {
+		repo, digest, _ := splitReference(mirror(n))
+		listing.repo(repo).Revisions[digest] = true
+		door.holds[mirror(n)] = true
+	}
+	for n := 1; n <= 7; n++ {
+		e := delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: "letta", FP: fp(n), Repository: "novox/x", Commit: "c",
+			Node: "ace", At: day.Add(time.Duration(n) * time.Hour), Outcome: "done"}
+		switch n {
+		case 2:
+			e.Against = []string{mirror(1)}
+		case 7:
+			e.Against = []string{mirror(2)}
+		}
+		raw, _ := delivery.EncodeBuildLogEntry(e)
+		b.log = append(b.log, stdio.LogEntry{Sequence: uint64(n), Key: "letta", Value: raw, At: e.At})
+	}
+	manifest, _ := delivery.EncodeManifest(delivery.Manifest{Module: "letta",
+		JSON: `{"module":"letta","resources":[{"id":"app","type":"container","image":"` + pinnedImage + `"},` +
+			`{"id":"built","type":"container","image":"${artifact:image}"}]}`})
+	mk, _ := delivery.ManifestsKey(strings.Repeat("d", 64))
+	_ = b.Put(ManifestsRead, mk, json.RawMessage(manifest))
+	want, _ := json.Marshal(delivery.Want{V: delivery.Version, FP: fp(1), Self: true})
+	_ = b.Put(WantsRead, "want.ace.letta", json.RawMessage(want))
+	if lease != nil {
+		_ = b.Put(LeaseRead, buscompose.ControllerLeaseKey, lease)
+	}
+	var acts []string
+	h := &Holder{Bus: b, Door: func() (Door, error) { return door, nil },
+		Listed: func(context.Context) (*View, error) { return &View{L: listing}, nil },
+		Now:    func() time.Time { return day.Add(48 * time.Hour) },
+		Acts: handacts.Recorder{Ask: func(key string, body any) (json.RawMessage, error) {
+			raw, _ := json.Marshal(body)
+			acts = append(acts, key+" "+string(raw))
+			return json.RawMessage(`{}`), nil
+		}}}
+	h.Acts.LeaseAway = h.LeaseAway
+	return h, b, door, &acts
+}
+
+// ADR 0571 clause 2: the keep set from the store's listing, the wants and the five newest builds; nothing let go of while
+// the lease names a controller; once it names another holder, a build's end lets go of the eligible within the bound,
+// leaves an index for a confirmed collect, records each deletion and says artifacts-gone.
+func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsOnlyOnceTheLeaseIsAway(t *testing.T) {
+	controller := &buscompose.LeaseHolder{Instance: "c1", Host: "novox", Epoch: 3}
+	handed := &buscompose.LeaseHolder{HandedTo: "mesh-grants", HandedBy: "person.jochen", Why: "the lease write"}
+
+	h, _, _, _ := aMesh(t, controller)
+	r, err := h.Artifacts("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	states := map[string]string{}
+	for _, x := range r.References {
+		states[x.Reference] = x.State
+	}
+	// letta's 3rd to 7th are the five newest; its 1st a want names on ace, however old; its 2nd is eligible; the package
+	// and the manifest no build names are no reference of the seat's.
+	for n, want := range map[int]string{1: "kept", 2: "eligible", 3: "kept", 7: "kept"} {
+		if states[imageRef("letta", n)] != want || states[archiveRef("letta", n)] != want {
+			t.Errorf("letta's build %d is %s and %s, not %s", n, states[imageRef("letta", n)], states[archiveRef("letta", n)], want)
+		}
+	}
+	// And the copies: the one letta's 2nd stood on is eligible with it, the 7th's kept; the third no build names.
+	if states[mirror(1)] != "eligible" || states[mirror(2)] != "kept" || states[mirror(3)] != "" {
+		t.Errorf("the copies are %s, %s and %q", states[mirror(1)], states[mirror(2)], states[mirror(3)])
+	}
+	if m, err := h.MirrorsNow(); err != nil || len(m.Copies) != 3 || m.Copies[2].State != "unrecorded" || m.Held != "" {
+		t.Errorf("mirrors answered %+v: %v", m, err)
+	}
+	if len(r.References) != 16 || r.Counts["kept"] != 13 || r.Counts["eligible"] != 3 || r.Store != "10.0.0.5:5000" {
+		t.Errorf("the references are %d, counted %v, at %q", len(r.References), r.Counts, r.Store)
+	}
+
+	for _, c := range []struct {
+		name    string
+		lease   *buscompose.LeaseHolder
+		index   bool
+		most    int
+		letGo   []string
+		left    int
+		skipped int
+		unlog   int
+	}{
+		{name: "the lease names a controller", lease: controller},
+		{name: "no lease key", lease: nil},
+		{name: "handed away", lease: handed, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2), mirror(1)}},
+		{name: "an index waits", lease: handed, index: true, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), mirror(1)}, skipped: 1},
+		{name: "bounded", lease: handed, most: 1, letGo: []string{archiveRef("letta", 2)}, left: 2},
+		// A kept build the log no longer holds: what it stood on is unknown, so no copy goes.
+		{name: "a kept build's bases unknown", lease: handed, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2)}, unlog: 3},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			h, b, door, acts := aMesh(t, c.lease)
+			door.indexes[imageRef("letta", 2)] = c.index
+			if c.unlog > 0 {
+				b.log = append(b.log[:c.unlog-1:c.unlog-1], b.log[c.unlog:]...)
+			}
+			if c.most == 0 {
+				h.AfterBuild("letta.x")
+				if _, err := h.Collect("tidy", true, 0); err == nil {
+					t.Error("a confirmed collect was let through while the controller sweeps")
+				}
+				if len(door.asked) != 0 || len(*acts) != 0 || len(b.published) != 0 {
+					t.Fatalf("before the lease write the store was asked %v, acts %v, said %v", door.asked, *acts, b.published)
+				}
+				return
+			}
+			a, err := h.Sweep(context.Background(), Sweeping{Most: c.most, By: "after the build letta.x"})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if fmt.Sprint(a.LetGo) != fmt.Sprint(c.letGo) || a.Left != c.left || a.Skipped != c.skipped || a.Held != 6 || a.Missing != 0 {
+				t.Fatalf("let go of %v (left %d, skipped %d, held %d, missing %d), want %v", a.LetGo, a.Left, a.Skipped, a.Held, a.Missing, c.letGo)
+			}
+			// Every kept archive held before the first deletion.
+			first := -1
+			for i, x := range door.asked {
+				if strings.HasPrefix(x, "let go ") && first < 0 {
+					first = i
+				}
+			}
+			if first != 6 {
+				t.Errorf("the store was asked %v: the six kept archives are held first", door.asked)
+			}
+			if len(*acts) != len(c.letGo) || !strings.HasPrefix((*acts)[0], "seat:mesh-hand-acts.hand-act ") ||
+				!strings.Contains((*acts)[0], c.letGo[0]) {
+				t.Errorf("the hand acts recorded are %v", *acts)
+			}
+			if len(b.published) != 1 || !strings.HasPrefix(b.published[0], GoneSubject+" ") ||
+				!strings.Contains(b.published[0], `"fingerprint":"`+fp(2)+`"`) {
+				t.Errorf("said %v", b.published)
+			}
+		})
+	}
+}
+
+// S25 (image-missing): a want whose build names an image the store does not hold is a finding in the seat's own state,
+// written once and deleted once the store holds it again; images names what a machine's wants and haves name.
+func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testing.T) {
+	h, b, _, _ := aMesh(t, nil)
+	view, _ := h.Listed(context.Background())
+	repo, digest, _ := splitReference(imageRef("letta", 1))
+	delete(view.L.Repos[repo].Revisions, digest)
+	h.Listed = func(context.Context) (*View, error) { return view, nil }
+	if err := h.Observe(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	key := health.FindingKey(health.ScopeModule, "letta.ace", ImageMissing)
+	raw, found, _ := b.Get(FindingsState, key)
+	f, err := health.DecodeFinding(raw)
+	if !found || err != nil || f.Subject.Machine != "ace" || f.Source != "S25" || !strings.Contains(f.Evidence[0].Said, imageRef("letta", 1)) {
+		t.Fatalf("the finding is %+v (%v, %v)", f, found, err)
+	}
+	view.L.Repos[repo].Revisions[digest] = true
+	if err := h.Observe(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if _, found, _ := b.Get(FindingsState, key); found {
+		t.Fatal("the finding stayed once the store held the image again")
+	}
+	// An image the manifest pins by itself, under the store's address, is judged as the controller judged it.
+	delete(view.L.Repos["letta/pinned"].Revisions, "sha256:"+strings.Repeat("c", 64))
+	if err := h.Observe(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if raw, found, _ := b.Get(FindingsState, key); !found || !strings.Contains(string(raw), "letta/pinned@sha256:") {
+		t.Fatalf("the pinnedImage image missing is %s", raw)
+	}
+
+	have, _ := json.Marshal(delivery.Have{V: delivery.Version, FP: fp(7), Phase: "running"})
+	_ = b.Put(HavesRead, "ace.letta", json.RawMessage(have))
+	images, err := h.ImagesOf("ace", "10.0.0.5:5000")
+	if err != nil || len(images.Images) != 3 || !images.SentKnown || images.Images[0].Image != "10.0.0.5:5000/letta/image@sha256:"+strings.Repeat("1", 64) ||
+		fmt.Sprint(images.Images[0].In) != "[declared]" || fmt.Sprint(images.Images[1].In) != "[sent]" ||
+		images.Images[2].Image != pinnedImage || fmt.Sprint(images.Images[2].In) != "[declared sent]" {
+		t.Fatalf("the images of ace are %+v: %v", images, err)
+	}
+}

modules/artifact-store-tools/cmd/artifact-store-tools/main.go

@@ -1,30 +1,38 @@
-// The artifact-store-tools module's Go bundle (novox/hq ADR 0251 §1–3, to-be 51): a process the node's
-// runtime launches and speaks MCP over stdio to. It says what the artifact store holds — its
-// repositories, its size, and what the controller's records say of each manifest — read from the
-// store's own files through the store's own container, and writing nothing. A collection is asked of
-// the controller, which decides and records what it lets go of (ADR 0189).
+// The artifact-store-tools module's Go bundle (novox/hq ADR 0251 §1–3, ADR 0571 clause 2, to-be 51): a process the
+// node's runtime launches and speaks MCP over stdio to. It says what the artifact store holds — its repositories, its
+// size, and what the mesh keeps of each manifest — read from the store's own files through the store's own container,
+// and it holds the seat mesh-artifact-sweep: the keep set, the sweep at each build's end once the controller's lease names
+// no controller, and S25's image-missing.
 //
-// **A module beside the store, not the store's own.** A module that provides the artifact store cannot
-// also build an artifact: building publishes to the store, so the store would be needed to create
-// itself, and the controller refuses that manifest. So these tools are a second module, assigned to the
-// machine that holds the store. stdout is the protocol; what this bundle says, it says on stderr.
+// **A module beside the store, not the store's own.** A module that provides the artifact store cannot also build an
+// artifact: building publishes to the store, so the store would be needed to create itself. So these tools are a
+// second module, assigned to the machine that holds the store. stdout is the protocol; what this bundle says, it says
+// on stderr.
 package main
 
 import (
 	"fmt"
 	"os"
+	"time"
 
 	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/handacts"
 )
 
+func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
+
 func main() {
 	s := &Store{
 		Container: os.Getenv("MESH_ARTIFACT_STORE_CONTAINER"),
 		Run:       ExecRunner,
 		UID:       os.Getuid(),
 	}
+	h := &Holder{Bus: runtimeBus{}, Door: DoorAt(os.Getenv("MESH_ARTIFACT_STORE_ADDRESS_FILE")), Listed: s.View,
+		Now: time.Now, Logf: logf}
+	h.Acts = handacts.Recorder{Ask: stdio.Ask, LeaseAway: h.LeaseAway}
+	go follow(h)
 	// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): artifact-store-tools.
-	if err := stdio.Serve("", Tools(s, Controller{Ask: stdio.Ask}, &Catalogue{Ask: stdio.Ask})); err != nil {
+	if err := stdio.Serve("", append(Tools(s, h, &Catalogue{Ask: stdio.Ask}), SeatTools(h, s)...)); err != nil {
 		fmt.Fprintln(os.Stderr, err)
 		os.Exit(1)
 	}

modules/artifact-store-tools/cmd/artifact-store-tools/seat.go

@@ -0,0 +1,122 @@
+package main
+
+// The seat mesh-artifact-sweep's verbs (novox/hq ADR 0571 clause 2; catalogue.ArtifactSweepSeat), each a tool named
+// `artifact-store.<verb>` as the runtime serves a seat's verb, answering as the controller's verb answered: {output,
+// ok, answer}.
+
+import (
+	"fmt"
+	"strings"
+
+	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+)
+
+// verbAnswer is the controller's verbs' answer shape, which the builder and docker read (answerOf).
+type verbAnswer struct {
+	Output string `json:"output"`
+	OK     bool   `json:"ok"`
+	Answer any    `json:"answer"`
+}
+
+// SeatTools are the seat's verbs over h; s reads the store's listing for mirrors.
+func SeatTools(h *Holder, s *Store) []stdio.Tool {
+	seat := catalogue.ArtifactSweepSeat + "."
+	return []stdio.Tool{
+		{Name: seat + "artifacts", Description: "Every artifact the mesh made that the store holds: kept and why, eligible, " +
+			"or with collected those it no longer holds.",
+			Run: func(args map[string]any) (any, error) {
+				r, err := h.Artifacts(text(args, "repository"))
+				if err != nil {
+					return nil, err
+				}
+				if text(args, "collected") != "true" {
+					refs := r.References[:0:0]
+					for _, x := range r.References {
+						if x.State != "collected" {
+							refs = append(refs, x)
+						}
+					}
+					r.References = refs
+				}
+				return verbAnswer{Output: fmt.Sprintf("%d kept, %d eligible, %d collected\n", r.Counts["kept"],
+					r.Counts["eligible"], r.Counts["collected"]), OK: true, Answer: r}, nil
+			}},
+		{Name: seat + "held", Description: "Whether the store holds each reference given: held, gone or unknown with why.",
+			Run: func(args map[string]any) (any, error) {
+				a, err := h.Held(listOf(args["references"]))
+				if err != nil {
+					return nil, err
+				}
+				return verbAnswer{Output: fmt.Sprintf("the artifact store at %s holds %d of the %d reference(s) asked\n",
+					a.Store, len(a.Held), len(a.Held)+len(a.Gone)+len(a.Unknown)), OK: true, Answer: a}, nil
+			}},
+		{Name: seat + "collect", Description: "The sweep now: a dry run without confirm; with confirm and why, every kept " +
+			"archive held, then each eligible artifact let go of, each recorded in the hand-act log.",
+			Run: func(args map[string]any) (any, error) {
+				most, err := count(args, "most", MostPerSweep, 1, MostAtAll)
+				if err != nil {
+					return nil, err
+				}
+				a, err := h.Collect(text(args, "why"), text(args, "confirm") == "true", most)
+				if err != nil {
+					return nil, err
+				}
+				return verbAnswer{Output: fmt.Sprintf("let go of %d, would let go of %d, %d left\n", len(a.LetGo),
+					len(a.WouldLetGo), a.Left), OK: true, Answer: a}, nil
+			}},
+		{Name: seat + "mirrors", Description: "The bases the mesh copied into its store, each kept (a kept build stood on " +
+			"it), eligible (only builds no longer kept stood on it) or unrecorded (no build the builder's log holds stood " +
+			"on it: never removed).",
+			Run: func(args map[string]any) (any, error) {
+				if text(args, "record") != "" {
+					return nil, fmt.Errorf("mirrors records nothing by hand on this seat: a copy is recorded by the builds the " +
+						"builder's log says stood on it (novox/hq ADR 0571 clause 1)")
+				}
+				m, err := h.MirrorsNow()
+				if err != nil {
+					return nil, err
+				}
+				return verbAnswer{Output: fmt.Sprintf("%d copies\n", len(m.Copies)), OK: true, Answer: m}, nil
+			}},
+		{Name: seat + "images", Description: "Every container image one machine's wants and haves name.",
+			Run: func(args map[string]any) (any, error) {
+				node := text(args, "node")
+				if node == "" {
+					return nil, fmt.Errorf("images names the machine")
+				}
+				address := ""
+				if d, err := h.Door(); err == nil {
+					address = d.Address()
+				}
+				a, err := h.ImagesOf(node, address)
+				if err != nil {
+					return nil, err
+				}
+				return verbAnswer{Output: fmt.Sprintf("%s: %d images\n", node, len(a.Images)), OK: true, Answer: a}, nil
+			}},
+	}
+}
+
+// listOf is a list argument: a list, or one text separated by commas.
+func listOf(v any) []string {
+	var out []string
+	add := func(s string) {
+		for _, p := range strings.Split(s, ",") {
+			if p = strings.TrimSpace(p); p != "" && !contains(out, p) {
+				out = append(out, p)
+			}
+		}
+	}
+	switch x := v.(type) {
+	case string:
+		add(x)
+	case []any:
+		for _, e := range x {
+			if s, ok := e.(string); ok {
+				add(s)
+			}
+		}
+	}
+	return out
+}

modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+
 	"context"
 	"encoding/json"
 	"fmt"
@@ -411,6 +413,11 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
 	var m struct {
 		Tools   []string `json:"tools"`
 		Invokes []string `json:"invokes"`
+		Reads   []string `json:"reads"`
+		Claims  []struct {
+			Name   string   `json:"name"`
+			Serves []string `json:"serves"`
+		} `json:"claims"`
 	}
 	if err := json.Unmarshal(raw, &m); err != nil {
 		t.Fatal(err)
@@ -428,10 +435,30 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
 	if !reflect.DeepEqual(served, listed) {
 		t.Fatalf("served %v, manifest %v", served, listed)
 	}
-	want := []string{"seat:mesh-controller.artifacts", "seat:mesh-controller.collect", CatalogueModules, CatalogueModule}
+	// ADR 0571 clause 2: it holds the seat mesh-artifact-sweep and serves its verbs, and calls no verb of the controller's;
+	// the lease is the one controller state it reads (clause 10's exemption).
+	want := []string{"seat:mesh-hand-acts.hand-act", CatalogueModules, CatalogueModule}
 	if !reflect.DeepEqual(m.Invokes, want) {
 		t.Errorf("invokes %v", m.Invokes)
 	}
+	var verbs []string
+	for _, tl := range SeatTools(&Holder{}, &Store{}) {
+		verbs = append(verbs, strings.TrimPrefix(tl.Name, catalogue.ArtifactSweepSeat+"."))
+	}
+	seat, _ := catalogue.SeatNamed(catalogue.ArtifactSweepSeat)
+	var contract []string
+	for _, v := range seat.Serves {
+		contract = append(contract, v.Name)
+	}
+	if len(m.Claims) != 1 || m.Claims[0].Name != catalogue.ArtifactSweepSeat || !reflect.DeepEqual(m.Claims[0].Serves, verbs) ||
+		!reflect.DeepEqual(verbs, contract) {
+		t.Errorf("claims %+v, serves %v, the seat's verbs %v", m.Claims, verbs, contract)
+	}
+	for _, r := range m.Reads {
+		if strings.HasPrefix(r, "mesh-controller.") && r != LeaseRead {
+			t.Errorf("reads the controller's %s", r)
+		}
+	}
 }
 
 func TestASocketRefusalIsAskedAgainThroughSudo(t *testing.T) {

modules/artifact-store-tools/cmd/artifact-store-tools/tools.go

@@ -11,9 +11,16 @@ import (
 	stdio "git.novox.be/novox/mesh-sdk/go"
 )
 
+// Recorded is what says which references the mesh keeps and lets go of them: the seat's holder (Holder, novox/hq ADR
+// 0571 clause 2), or before it the controller (Controller), each in the controller's answer's shape.
+type Recorded interface {
+	Artifacts(repository string) (*Records, error)
+	Collect(why string, confirm bool, most int) (*CollectAnswer, error)
+}
+
 // Tools are the artifact store's tools (novox/hq ADR 0251 §1). The first three only read; artifact_store_collect
-// only reads too, unless it is a real run, and then it asks the controller, which decides and deletes.
-func Tools(s *Store, c Controller, cat *Catalogue) []stdio.Tool {
+// only reads too, unless it is a real run, and then it asks the seat's sweep, which decides and deletes.
+func Tools(s *Store, c Recorded, cat *Catalogue) []stdio.Tool {
 	ctx := context.Background()
 	repoArg := map[string]any{"type": "string", "description": "one repository, as the store names it (<module>/<artifact>)"}
 	return []stdio.Tool{