Node shadow: reuse a running seal only at its own slot, lay values as the controller does, answer the newest candidate first and say how it goes (issue 1126, C1 C2, T1 T2 T3)

A box is reused by slot (resource, field) after the composition, from what runs or the
have's slots; a have written before slots is reused within its module alone. Config
values are laid by mesh-sdk's one rule (never a default). The shadow answers the
newest candidate first and hands each answer as it lands, says when an answer starts,
every minute what it waits on, and when it ends, and fits an answer to the node's key.
Needs mesh-sdk fix/1126-seed-same (16b8d8a1).

cmd/mesh-host/nodeagent.go

@@ -62,14 +62,14 @@ func (h sessionHaves) Read(ctx context.Context, module string) (agent.Have, bool
 		if !ok || got.Composed == "" {
 			return agent.Have{}, false
 		}
-		return agent.Have{Module: module, Composed: got.Composed, Sealed: got.Sealed}, true
+		return agent.Have{Module: module, Composed: got.Composed, Sealed: got.SealedAt}, true
 	}
 	got, ok := h.session.ReadHave(ctx, module)
 	if !ok {
 		return agent.Have{}, false
 	}
 	return agent.Have{Module: module, FP: got.FP, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail,
-		Sealed: got.Sealed}, true
+		Sealed: got.SealedAt}, true
 }
 
 // ReadErr is Read telling a have absent from one the bus could not read (agent.HavesReadErr).
@@ -83,7 +83,7 @@ func (h sessionHaves) ReadErr(ctx context.Context, module string) (agent.Have, b
 		return agent.Have{}, false, err
 	}
 	return agent.Have{Module: module, FP: got.FP, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail,
-		Sealed: got.Sealed}, true, nil
+		Sealed: got.SealedAt}, true, nil
 }
 
 func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
@@ -535,16 +535,16 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
 			},
 			unseal: sealOpener(opts.state),
 			// And what each module's have says it runs sealed: after a restart the agent has applied nothing yet.
-			held: func(module string) []string {
+			held: func(module string) (delivery.Seals, []string) {
 				if module == agent.NodeModule {
 					got, _ := session.ReadNodeHave(linked)
-					return got.Sealed
+					return got.SealedAt, got.Sealed
 				}
 				got, ok := session.ReadHave(linked, module)
 				if !ok {
-					return nil
+					return nil, nil
 				}
-				return got.Sealed
+				return got.SealedAt, got.Sealed
 			},
 			refused: func(why string) {
 				if now := own.get(); now != nil && run.Err() == nil {
@@ -870,7 +870,7 @@ type composerHooks struct {
 	running func() []delivery.Composed
 	unseal  func(string) ([]byte, error)
 	refused func(string)
-	held    func(module string) []string
+	held    func(module string) (delivery.Seals, []string)
 }
 
 func selfComposer(ctx context.Context, session *link.Session, node string, say func(string), hooks composerHooks) (*selfcompose.Composer, string) {

internal/agent/agent.go

@@ -57,9 +57,10 @@ type Have struct {
 	// waits for a person's yes (ADR 0547 §3).
 	Judgement string
 	Waits     string
-	// Sealed is, on a node composing its own, each value the composition it names carries sealed to the node's key
-	// (delivery.Have.Sealed): kept in the have so the composer reuses them after a restart (ADR 0547 §3).
-	Sealed []string
+	// Sealed is, on a node composing its own, each value the composition it names carries sealed to the node's key, by
+	// slot (delivery.Have.SealedAt): kept in the have so the composer reuses them after a restart (ADR 0547 §3), each at
+	// its own slot alone (issue 1126, C1).
+	Sealed delivery.Seals
 }
 
 // Haves is the node's have bucket as the agent uses it.
@@ -471,15 +472,15 @@ func (a *Agent) writeHave(ctx context.Context, h Have) {
 	a.env.Haves.Write(ctx, h)
 }
 
-func (a *Agent) sealedOf(ctx context.Context, module, fp, composed string) []string {
+func (a *Agent) sealedOf(ctx context.Context, module, fp, composed string) delivery.Seals {
 	a.mu.Lock()
 	if c, ok := a.applied[module]; ok && c.Err == nil && c.Dig == composed {
 		a.mu.Unlock()
-		return SealedIn(c.C)
+		return delivery.SealsIn(c.C)
 	}
 	if c, ok := a.compositionFor(module, fp); ok && c.Err == nil && c.Dig == composed {
 		a.mu.Unlock()
-		return SealedIn(c.C)
+		return delivery.SealsIn(c.C)
 	}
 	a.mu.Unlock()
 	if h, ok := a.env.Haves.Read(ctx, module); ok && h.Composed == composed {
@@ -488,32 +489,6 @@ func (a *Agent) sealedOf(ctx context.Context, module, fp, composed string) []str
 	return nil
 }
 
-// SealedIn is each value a composition carries sealed to its node's key: a file's `sealed` and a vault box
-// (catalogue.VaultSealedField), as their JSON text, in the order of its resources.
-func SealedIn(c delivery.Composed) []string {
-	var out []string
-	for _, raw := range c.Resources {
-		var r map[string]json.RawMessage
-		if json.Unmarshal(raw, &r) != nil {
-			continue
-		}
-		for _, field := range []string{"sealed", catalogue.VaultSealedField} {
-			f, ok := r[field]
-			if !ok {
-				continue
-			}
-			var text string
-			if json.Unmarshal(f, &text) != nil {
-				text = string(f) // a vault box is carried as its JSON
-			}
-			if text != "" {
-				out = append(out, text)
-			}
-		}
-	}
-	return out
-}
-
 // Running is the compositions the node runs now: each module's that the agent applied or found already running, and
 // the node's own. What the node judges its health by and says of itself is built from them.
 func (a *Agent) Running() []delivery.Composed {

internal/agent/agent_test.go

@@ -5,6 +5,7 @@ import (
 	"crypto/ed25519"
 	"encoding/json"
 	"fmt"
+	"maps"
 	"slices"
 	"strings"
 	"sync"
@@ -1064,30 +1065,30 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
 	b.wants.put(nwk, nwraw)
 	eventually(t, "the node's own key does not carry its sealed values", func() bool {
 		h := b.haves.last(NodeModule)
-		return h.Composed == ndig && len(h.Sealed) == 1 && h.Sealed[0] == "core"
+		return h.Composed == ndig && len(h.Sealed) == 1 && h.Sealed["core-box#sealed"] == "core"
 	})
 	// The node cannot compose (a stand-in refused): each wanted module's have says so, failed, naming why, and still
 	// names what runs; composed again, the have is what it runs. web was applied by this agent; idle was not since the
 	// agent's start (the switch and every delivery of the node-engine restart it), so what runs is what its have said.
 	idleFP, idleDig := fpOf("c"), "sha256:"+strings.Repeat("9", 64)
 	b.haves.Write(context.Background(), Have{Module: "idle", FP: idleFP, Composed: idleDig, Phase: string(delivery.PhaseRunning),
-		Sealed: []string{`{"box":"idle"}`}})
+		Sealed: delivery.Seals{"idle.key#sealed": `{"box":"idle"}`}})
 	want("idle", idleFP, "")
 	eventually(t, "idle's want was not taken", func() bool {
 		b.ag.mu.Lock()
 		defer b.ag.mu.Unlock()
 		return b.ag.wants["idle"].Present
 	})
-	if h := b.haves.last("web"); len(h.Sealed) != 1 || h.Sealed[0] != "1" {
+	if h := b.haves.last("web"); len(h.Sealed) != 1 || h.Sealed["web.pw#sealed"] != "1" {
 		t.Errorf("web's running have does not carry its sealed value: %+v", h)
 	}
 	b.ag.NotComposed(context.Background(), "a stand-in for web's password")
 	for _, row := range []struct {
-		module, fp, composed, sealed string
-	}{{"web", fp, dig, "1"}, {"idle", idleFP, idleDig, `{"box":"idle"}`}} {
+		module, fp, composed, slot, sealed string
+	}{{"web", fp, dig, "web.pw#sealed", "1"}, {"idle", idleFP, idleDig, "idle.key#sealed", `{"box":"idle"}`}} {
 		h := b.haves.last(row.module)
 		if h.Phase != string(delivery.PhaseFailed) || h.FP != row.fp || h.Composed != row.composed ||
-			len(h.Sealed) != 1 || h.Sealed[0] != row.sealed || !strings.Contains(h.Detail, "web's password") {
+			len(h.Sealed) != 1 || h.Sealed[row.slot] != row.sealed || !strings.Contains(h.Detail, "web's password") {
 			t.Errorf("%s's have while the node cannot compose: %+v", row.module, h)
 		}
 	}
@@ -1155,7 +1156,7 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
 	// what runs (fingerprint, digest, sealed values) rather than naming nothing (review of L2, round 3).
 	recFP, recDig := fpOf("d"), "sha256:"+strings.Repeat("8", 64)
 	b.haves.Write(context.Background(), Have{Module: "rec", FP: recFP, Composed: recDig, Phase: string(delivery.PhaseRunning),
-		Sealed: []string{`{"box":"rec"}`}})
+		Sealed: delivery.Seals{"rec.key#sealed": `{"box":"rec"}`}})
 	rk, rraw, rdig := unsigned("rec", fpB, container("rec@sha256:"+strings.Repeat("3", 64)))
 	b.comps.put(dk2, draw2)
 	b.comps.put(rk, rraw)
@@ -1164,15 +1165,15 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
 	applied := len(b.appliedList())
 	for _, row := range []struct {
 		module, fp, composed, waits string
-		sealed                      []string
-	}{{"db", fp, "", ddig2, nil}, {"rec", recFP, recDig, rdig, []string{`{"box":"rec"}`}}} {
+		sealed                      delivery.Seals
+	}{{"db", fp, "", ddig2, nil}, {"rec", recFP, recDig, rdig, delivery.Seals{"rec.key#sealed": `{"box":"rec"}`}}} {
 		eventually(t, row.module+"'s move did not wait for a yes", func() bool {
 			h := b.haves.last(row.module)
 			return h.Phase == string(delivery.PhaseWaitingYes) && h.Waits == row.waits
 		})
 		h := b.haves.last(row.module)
 		if h.FP != row.fp || (row.composed != "" && h.Composed != row.composed) || h.Composed == "" ||
-			(row.sealed != nil && !slices.Equal(h.Sealed, row.sealed)) {
+			(row.sealed != nil && !maps.Equal(h.Sealed, row.sealed)) {
 			t.Errorf("%s's have waiting for a yes does not keep what runs: %+v", row.module, h)
 		}
 	}

internal/link/have.go

@@ -49,7 +49,7 @@ type haveState struct {
 	nodeComposed string
 	// nodeSealed is, on a node composing its own, each value its own composition carries sealed to its key (ADR 0547 §3,
 	// issue 1126): kept in its key so a restart reuses them and applies nothing again.
-	nodeSealed []string
+	nodeSealed delivery.Seals
 	// nodeCandidates is the node's answer to each placement candidate group (novox/hq ADR 0559 §4), written into its key.
 	nodeCandidates map[string]delivery.CandidateAnswer
 	// nodeComposes and nodeComposesWhy are whose compositions the node applies and why (HaveNode.Composes), written into
@@ -383,7 +383,7 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
 		case f.Failed:
 			h.Phase = delivery.PhaseFailed
 		}
-		h.Sealed = f.Sealed
+		h.SealedAt = f.Sealed
 		h.Health = s.healthOf(module, f)
 		if h.Detail == "" {
 			h.Detail = s.wordsOf(module, h.Health)
@@ -424,7 +424,7 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
 			var raw []byte
 			facts := *s.node
 			facts.Composed = s.nodeComposed
-			facts.Sealed = s.nodeSealed
+			facts.SealedAt = s.nodeSealed
 			facts.Candidates = s.nodeCandidates
 			facts.Composes, facts.ComposesWhy = s.nodeComposes, s.nodeComposesWhy
 			if raw, err = delivery.EncodeHaveNode(facts); err == nil {
@@ -716,7 +716,7 @@ func (q *Session) NoteRemoved(ctx context.Context, module, build string) {
 
 // NoteNode takes the digest of the node's own composition it runs and the values it carries sealed to the node's key
 // (nil for a composition the node did not make itself), and writes the node's key where it differs.
-func (q *Session) NoteNode(ctx context.Context, composed string, sealed []string) {
+func (q *Session) NoteNode(ctx context.Context, composed string, sealed delivery.Seals) {
 	q.init()
 	s := &q.haves
 	s.mu.Lock()

internal/link/have_test.go

@@ -582,11 +582,11 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth(
 	if got := l.have(t, "n1.web"); got.Phase != delivery.PhaseRemoved {
 		t.Fatalf("removed: %+v", got)
 	}
-	q.NoteNode(context.Background(), dig, []string{"core"})
-	if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Composed != dig || len(n.Sealed) != 1 || n.Sealed[0] != "core" {
-		t.Fatalf("the node key, with its own composition's sealed values: %+v %v", n, err)
+	q.NoteNode(context.Background(), dig, delivery.Seals{"core-box#sealed": "core"})
+	if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Composed != dig || len(n.SealedAt) != 1 || n.SealedAt["core-box#sealed"] != "core" {
+		t.Fatalf("the node key, with its own composition's sealed values by slot: %+v %v", n, err)
 	}
-	if n, ok := q.ReadNodeHave(context.Background()); !ok || n.Composed != dig || len(n.Sealed) != 1 {
+	if n, ok := q.ReadNodeHave(context.Background()); !ok || n.Composed != dig || len(n.SealedAt) != 1 {
 		t.Fatalf("the node key read back: %+v %v", n, ok)
 	}
 	// The node's answers to placement candidates ride in the same key (ADR 0559 §4), and go when no group is answered.

internal/link/messages.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 	"time"
 
+	"git.novox.be/novox/mesh-sdk/go/delivery"
 	"git.novox.be/novox/mesh-sdk/go/health"
 )
 
@@ -746,6 +747,6 @@ type ModuleFact struct {
 	// Judgement and Waits are, in phase waiting-yes, the judgement of the move and the composition waiting (ADR 0547 §3).
 	Judgement string
 	Waits     string
-	// Sealed is the values the composition it names carries sealed to this node's key (delivery.Have.Sealed).
-	Sealed []string
+	// Sealed is the values the composition it names carries sealed to this node's key, by slot (delivery.Have.SealedAt).
+	Sealed delivery.Seals
 }

internal/selfcompose/selfcompose.go

@@ -11,7 +11,6 @@
 package selfcompose
 
 import (
-	"bytes"
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
@@ -56,14 +55,18 @@ type Env struct {
 	// have says it (agent.NotComposed): a refusal is loud where haves are read, not one journal line. Nil tells nobody.
 	Refused func(why string)
 	// Running is what the node runs now (agent.Everything), and Unseal opens a value sealed to this machine: a secret
-	// the vault seals again for each ask is kept as the box the node already runs when that box opens to the same
-	// value, so neither the switch to composing its own nor a restart changes a composition (ADR 0547 §3). Either nil
-	// keeps the vault's box.
+	// the vault seals again for each ask is kept as the box the node already runs at the same slot (the same field of
+	// the same resource of the same module, delivery.KeepSeals) when that box opens to the same value, so neither the
+	// switch to composing its own nor a restart changes a composition (ADR 0547 §3); a box held at another slot is
+	// never taken, though it opens to the same value (issue 1126, C1: a provider's grant and its consumer's secret).
+	// Either nil keeps the vault's box.
 	Running func() []delivery.Composed
 	Unseal  func(sealed string) ([]byte, error)
-	// Held is the sealed values a module's have says it runs (delivery.Have.Sealed), kept by the agent on the bus: what
-	// runs after a restart, before the agent applied anything, and after the switch once the kept declaration is old.
-	Held func(module string) []string
+	// Held is the sealed values a module's have says it runs, kept by the agent on the bus: what runs after a restart,
+	// before the agent applied anything, and after the switch once the kept declaration is old. By slot
+	// (delivery.Have.SealedAt); legacy is the list a have written before slots holds (delivery.Have.Sealed), reused
+	// only within its own module.
+	Held func(module string) (slots delivery.Seals, legacy []string)
 	// Fetch reads an archive at a URL in the artifact store: a moved module's templates archive, whose texts its built
 	// manifest carries (issue 1126). Read once per digest. Nil reads none, and a moved module does not compose.
 	Fetch func(ctx context.Context, url string) ([]byte, error)
@@ -93,8 +96,12 @@ type Composer struct {
 	sealedOwn map[string]ownSealed
 	said      string
 	refusal   string
-	caught    bool
-	rev       uint64
+	// doing is what the composer waits on now (waiting), for a running answer to say (issue 1126, T2); its own lock, so
+	// it is read while c.mu is held.
+	doingMu sync.Mutex
+	doing   string
+	caught  bool
+	rev     uint64
 }
 
 // New is a composer for one node.
@@ -399,11 +406,10 @@ func (c *Composer) takeValue(u agent.Update) {
 	}
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	var r struct {
-		Value json.RawMessage `json:"value"`
-	}
-	var v any
-	if u.Deleted || json.Unmarshal(u.Value, &r) != nil || len(r.Value) == 0 || json.Unmarshal(r.Value, &v) != nil {
+	// Only a value its source lays: a default is the composition's already, as the controller reads them
+	// (catalogue.LaidValue, issue 1126, C2: Element's links blanked by a laid default).
+	v, laid := catalogue.LaidValue(u.Value)
+	if u.Deleted || !laid {
 		delete(c.values[module], key)
 		return
 	}
@@ -479,6 +485,10 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
 	}
 	c.refused("")
 	c.sayOnce("")
+	// The boxes it runs, slot by slot: read before c.mu is taken, since the agent's lock is taken to read what runs.
+	for key, comp := range compositions {
+		compositions[key] = c.keepRunning(comp, running)
+	}
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	var out []agent.Update
@@ -532,45 +542,69 @@ func (c *Composer) refused(why string) {
 	}
 }
 
-// runningBox is the box the node already runs that opens to the same value as box, or box: what the agent runs, and
-// what each wanted module's have says it runs sealed (Held). Read before c.mu is taken, since the agent's lock is taken
-// to read what runs.
-func (c *Composer) runningBox(box string) string {
-	if c.env.Unseal == nil || (c.env.Running == nil && c.env.Held == nil) {
-		return box
+// keepRunning is comp with each sealed value the node already runs at the same slot kept, where it opens to the same
+// value (delivery.KeepSeals, issue 1126, C1): of what runs (running, the agent's), else of what its module's have says
+// it runs (Held). A have written before slots names no slot: its boxes are reused within their own module alone, at a
+// slot no box is kept for, and only one opening to the slot's value.
+func (c *Composer) keepRunning(comp delivery.Composed, running []delivery.Composed) delivery.Composed {
+	if c.env.Unseal == nil {
+		return comp
 	}
-	value, err := c.env.Unseal(box)
-	if err != nil {
-		return box
-	}
-	var held []string
-	if c.env.Running != nil {
-		for _, comp := range c.env.Running() {
-			held = append(held, agent.SealedIn(comp)...)
-		}
+	module := comp.Module
+	if module == "" {
+		module = agent.NodeModule
 	}
+	kept := delivery.Seals{}
+	var legacy []string
 	if c.env.Held != nil {
-		c.mu.Lock()
-		// The node's own key among them (agent.NodeModule): its core boxes, as its last run said them.
-		modules := make([]string, 0, len(c.wants))
-		for m := range c.wants {
-			modules = append(modules, m)
+		slots, old := c.env.Held(module)
+		for slot, box := range slots {
+			kept[slot] = box
 		}
-		c.mu.Unlock()
-		sort.Strings(modules)
-		for _, m := range modules {
-			held = append(held, c.env.Held(m)...)
+		legacy = old
+	}
+	for _, r := range running {
+		if r.Module == comp.Module {
+			for slot, box := range delivery.SealsIn(r) {
+				kept[slot] = box
+			}
 		}
 	}
-	for _, h := range held {
-		if h == box {
+	for slot, box := range delivery.SealsIn(comp) {
+		if _, ok := kept[slot]; ok {
 			continue
 		}
-		if v, err := c.env.Unseal(h); err == nil && bytes.Equal(v, value) {
-			return h
+		for _, h := range legacy {
+			if same, known := c.sameValue(box, h); known && same {
+				kept[slot] = h
+				break
+			}
 		}
 	}
-	return box
+	return delivery.KeepSeals(comp, kept, c.sameValue)
+}
+
+// waiting notes what the composer waits on until the returned func is called, which restores what it waited on before.
+func (c *Composer) waiting(what string) func() {
+	c.doingMu.Lock()
+	before := c.doing
+	c.doing = what
+	c.doingMu.Unlock()
+	return func() {
+		c.doingMu.Lock()
+		c.doing = before
+		c.doingMu.Unlock()
+	}
+}
+
+// waitingOn is what the composer waits on now, in words.
+func (c *Composer) waitingOn() string {
+	c.doingMu.Lock()
+	defer c.doingMu.Unlock()
+	if c.doing == "" {
+		return "nothing it reads (composing)"
+	}
+	return c.doing
 }
 
 func (c *Composer) sayOnce(line string) {
@@ -652,6 +686,7 @@ func (r reader) Manifest(ctx context.Context, digest string) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
+	defer r.c.waiting("the manifest " + digest + " (" + delivery.ManifestsBucket + ")")()
 	value, err := r.c.env.Get(ctx, delivery.ManifestsBucket, key)
 	if err != nil {
 		return nil, err
@@ -674,6 +709,7 @@ func (r reader) Build(ctx context.Context, module, fp string) (*delivery.Builder
 	if err != nil {
 		return nil, err
 	}
+	defer r.c.waiting("the build of " + module + " at " + fp + " (" + delivery.BuilderBuildsBucket + ")")()
 	raw, err := r.c.env.Get(ctx, delivery.BuilderBuildsBucket, key)
 	if err != nil || raw == nil {
 		return nil, err
@@ -700,6 +736,7 @@ func (r reader) Archive(ctx context.Context, url, digest string) ([]byte, error)
 		return nil, fmt.Errorf("this node reads no archive (%s)", url)
 	}
 	// Bounded: a store that does not answer must not hold the composer or the node's answer (issue 1126).
+	defer r.c.waiting("the archive " + url)()
 	fetching, cancel := context.WithTimeout(ctx, archiveWithin)
 	defer cancel()
 	body, err := r.c.env.Fetch(fetching, url)
@@ -719,17 +756,15 @@ func (r reader) Archive(ctx context.Context, url, digest string) ([]byte, error)
 // Served reads, of another node's module, the keys named of mesh-config's resolved values, one key each: the node is
 // granted those keys alone (buscompose Principal.ServedReads, issue 1126).
 func (r reader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) {
+	defer r.c.waiting("what " + module + " serves on " + node + " (" + ResolvedBucket + ")")()
 	out := map[string]any{}
 	for _, k := range keys {
 		raw, err := r.c.env.Get(ctx, ResolvedBucket, node+"."+module+"."+k)
 		if err != nil {
 			return nil, err
 		}
-		var v struct {
-			Value json.RawMessage `json:"value"`
-		}
-		var value any
-		if raw == nil || json.Unmarshal(raw, &v) != nil || len(v.Value) == 0 || json.Unmarshal(v.Value, &value) != nil {
+		value, laid := catalogue.LaidValue(raw)
+		if raw == nil || !laid {
 			continue
 		}
 		out[k] = value
@@ -760,7 +795,9 @@ func (r reader) OwnSecrets(ctx context.Context, node string) ([]compose.OwnSecre
 	if node != r.c.env.Node || r.c.env.Own == nil {
 		return nil, nil
 	}
+	done := r.c.waiting("the vault's own secrets")
 	raw, err := r.c.env.Own(ctx, map[string]any{})
+	done()
 	if err != nil {
 		return nil, fmt.Errorf("the vault's own: %w", err)
 	}
@@ -781,25 +818,16 @@ func (r reader) OwnSecrets(ctx context.Context, node string) ([]compose.OwnSecre
 	if a.Node != node {
 		return nil, fmt.Errorf("the vault answered %q's own secrets to %s", a.Node, node)
 	}
-	ids := make([]string, len(a.Own))
-	fresh := make([]string, len(a.Own))
-	for i, o := range a.Own {
-		ids[i] = o.Module + "/" + o.Name
-		if !r.c.keeps(ids[i], o.Generation, a.Key) {
-			fresh[i] = r.c.runningBox(string(o.Sealed))
-		}
-	}
 	r.c.mu.Lock()
 	defer r.c.mu.Unlock()
 	out := make([]compose.OwnSecretRow, 0, len(a.Own))
-	for i, o := range a.Own {
-		id := ids[i]
+	for _, o := range a.Own {
+		// The box first had is kept while its generation stands, so sealing it again moves no composition; the box the
+		// node runs at the same slot is kept after the composition (keepRunning).
+		id := o.Module + "/" + o.Name
 		kept, ok := r.c.sealedOwn[id]
 		if !ok || kept.generation != o.Generation || kept.key != a.Key {
-			kept = ownSealed{generation: o.Generation, key: a.Key, sealed: fresh[i]}
-			if kept.sealed == "" {
-				kept.sealed = string(o.Sealed)
-			}
+			kept = ownSealed{generation: o.Generation, key: a.Key, sealed: string(o.Sealed)}
 			r.c.sealedOwn[id] = kept
 		}
 		out = append(out, compose.OwnSecretRow{Node: node, Module: o.Module, Name: o.Name, Origin: o.Origin,
@@ -823,7 +851,9 @@ func (r reader) Credentials(ctx context.Context, node string, asConsumer, asProv
 	for _, k := range asProvider {
 		ask = append(ask, asked{Role: "provider", CredentialKey: k})
 	}
+	done := r.c.waiting("the vault's credentials")
 	raw, err := r.c.env.Own(ctx, map[string]any{"credentials": ask})
+	done()
 	if err != nil {
 		return nil, nil, fmt.Errorf("the vault's own: %w", err)
 	}
@@ -844,28 +874,17 @@ func (r reader) Credentials(ctx context.Context, node string, asConsumer, asProv
 	if a.Node != node {
 		return nil, nil, fmt.Errorf("the vault answered %q's credentials to %s", a.Node, node)
 	}
-	ids := make([]string, len(a.Credentials))
-	fresh := make([]string, len(a.Credentials))
-	for i, c := range a.Credentials {
-		ids[i] = c.Role + "|" + c.Name + "|" + c.Provision + "|" + c.Consumer + "|" + c.ConsumerModule + "|" + c.Local
-		if !r.c.keeps(ids[i], c.Generation, a.Key) {
-			fresh[i] = r.c.runningBox(string(c.Sealed))
-		}
-	}
 	r.c.mu.Lock()
 	defer r.c.mu.Unlock()
 	var creds []compose.Credential
 	var issued []compose.Issued
-	for i, c := range a.Credentials {
-		// The value first had is kept while its generation stands, so sealing it again moves no composition; and the
-		// first had is the box the node runs when it opens to the same value (runningBox).
-		id := ids[i]
+	for _, c := range a.Credentials {
+		// The value first had is kept while its generation stands, so sealing it again moves no composition; the box the
+		// node runs at the same slot is kept after the composition (keepRunning).
+		id := c.Role + "|" + c.Name + "|" + c.Provision + "|" + c.Consumer + "|" + c.ConsumerModule + "|" + c.Local
 		kept, ok := r.c.sealedOwn[id]
 		if !ok || kept.generation != c.Generation || kept.key != a.Key {
-			kept = ownSealed{generation: c.Generation, key: a.Key, sealed: fresh[i]}
-			if kept.sealed == "" {
-				kept.sealed = string(c.Sealed)
-			}
+			kept = ownSealed{generation: c.Generation, key: a.Key, sealed: string(c.Sealed)}
 			r.c.sealedOwn[id] = kept
 		}
 		k := c.CredentialKey
@@ -881,14 +900,6 @@ func (r reader) Credentials(ctx context.Context, node string, asConsumer, asProv
 	return creds, issued, nil
 }
 
-// keeps says the composer holds id's sealed value at this generation and key already.
-func (c *Composer) keeps(id string, generation int, key string) bool {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	kept, ok := c.sealedOwn[id]
-	return ok && kept.generation == generation && kept.key == key
-}
-
 // Placed reads whether the catalogue places node now, for the node agent choosing whose compositions it applies at
 // each start: true when its placement key holds a value, false only when a read succeeds and finds no key or a delete
 // marker. A read that fails chooses neither (review of the shadow, round 1: a timeout or a bus not ready at a restart

internal/selfcompose/selfcompose_test.go

@@ -306,8 +306,28 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 		t.Error("Transient misjudges a bus that does not answer, or a refusal")
 	}
 
+	// Issue 1126, C2: a config value is laid as the controller lays it (catalogue.LaidValue), from the mesh's, the node's
+	// or the module's own and never from a default or an unset one, both of this node's and of what another node serves:
+	// a default laid blanked Element's links on ace.
+	for _, row := range []struct {
+		source string
+		laid   bool
+	}{{"mesh", true}, {"node", true}, {"module", true}, {"default", false}, {"unset", false}, {"", false}} {
+		probe := New(env)
+		value := []byte(`{"value":["x"],"source":"` + row.source + `"}`)
+		probe.takeValue(agent.Update{Key: "n1.lemurs.links", Value: value})
+		_, laid := probe.values["lemurs"]["links"]
+		b.mu.Lock()
+		b.kv[ResolvedBucket+"/n2.lemurs.links"] = value
+		b.mu.Unlock()
+		served, err := reader{c: probe}.Served(ctx, "n2", "lemurs", []string{"links"})
+		if laid != row.laid || err != nil || (served["links"] != nil) != row.laid {
+			t.Errorf("a value from %q: laid %v, served %v (%v), want laid %v", row.source, laid, served, err, row.laid)
+		}
+	}
+
 	// A config value of another module, and a value changing nothing composed, hand nothing.
-	b.put(t, ResolvedBucket, "n1.other.k", []byte(`{"value":1}`))
+	b.put(t, ResolvedBucket, "n1.other.k", []byte(`{"value":1,"source":"node"}`))
 	quiet(t, out)
 
 	// A want removed takes its composition back.
@@ -367,7 +387,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 		t.Fatalf("keeper's composition does not carry the sealed value: %s", handed)
 	}
 	before := handed
-	b.put(t, ResolvedBucket, "n1.other.k", []byte(`{"value":2}`))
+	b.put(t, ResolvedBucket, "n1.other.k", []byte(`{"value":2,"source":"node"}`))
 	quiet(t, out)
 	b.mu.Lock()
 	generation++
@@ -383,32 +403,47 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 	// node runs comes from keeper's have (Held); the vault seals the value again, other bytes, and the box the have holds
 	// is kept, since it opens to the same value: the composition does not move. The box from before the rotation opens
 	// to the old value, so the vault's is taken.
-	boxOf := func(raw string) string {
+	// Issue 1126, C1: a box is kept only at the slot that holds it (the same field of the same resource of the same
+	// module). A box another module holds (a provider's grant of the value its consumer holds as a secret, on one node)
+	// or another resource of keeper holds is never taken, though it opens to the same value; a have written before slots
+	// (a list) is reused within its own module alone.
+	boxOf := func(raw string) (string, string) {
 		c, err := delivery.DecodeComposed([]byte(raw))
 		if err != nil {
 			t.Fatal(err)
 		}
-		sealed := agent.SealedIn(c)
+		sealed := delivery.SealsIn(c)
 		if len(sealed) != 1 {
 			t.Fatalf("keeper's composition carries %v", sealed)
 		}
-		return sealed[0]
+		for slot, box := range sealed {
+			return slot, box
+		}
+		return "", ""
 	}
-	// And the node's own key (review of L2, round 3, finding 1): a core box its own composition carries is held there, and
-	// kept after a restart as a module's is, so the node's own composition does not move either.
+	slot, rotatedBox := boxOf(string(rotated.Value))
+	_, beforeBox := boxOf(before)
 	for _, row := range []struct {
-		name, held, in string
+		name, in, held string
+		slots          delivery.Seals
 		kept           bool
-	}{{"the have holds the rotated value", boxOf(string(rotated.Value)), "keeper", true},
-		{"the have holds the value before the rotation", boxOf(before), "keeper", false},
-		{"the node's own key holds the rotated value", boxOf(string(rotated.Value)), agent.NodeModule, true}} {
+	}{
+		{"the have holds the rotated value at its slot", "keeper", rotatedBox, delivery.Seals{slot: rotatedBox}, true},
+		{"the have holds the value before the rotation", "keeper", beforeBox, delivery.Seals{slot: beforeBox}, false},
+		{"a have written before slots holds the rotated value", "keeper", rotatedBox, nil, true},
+		{"another module holds the same value", "lemurs", rotatedBox, delivery.Seals{"lemurs.grant-x#sealed": rotatedBox}, false},
+		{"another resource of keeper holds the same value", "keeper", rotatedBox, delivery.Seals{"keeper.other#sealed": rotatedBox}, false},
+	} {
 		restarted := env
 		restarted.Running = func() []delivery.Composed { return nil }
-		restarted.Held = func(module string) []string {
-			if module == row.in {
-				return []string{row.held}
+		restarted.Held = func(module string) (delivery.Seals, []string) {
+			if module != row.in {
+				return nil, nil
 			}
-			return nil
+			if row.slots == nil {
+				return nil, []string{row.held}
+			}
+			return row.slots, nil
 		}
 		restarted.Unseal = func(box string) ([]byte, error) {
 			b.mu.Lock()
@@ -419,14 +454,56 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 			return nil, fmt.Errorf("not sealed to n1: %s", box)
 		}
 		again := New(restarted)
-		again.wants = map[string]delivery.Want{"keeper": {FP: fp, Self: true}, agent.NodeModule: {}}
-		rows, err := reader{c: again}.OwnSecrets(ctx, "n1")
-		if err != nil || len(rows) != 1 {
-			t.Fatalf("%s: own secrets %+v, %v", row.name, rows, err)
+		c.mu.Lock()
+		for k, v := range c.wants {
+			again.wants[k] = v
 		}
-		if kept := rows[0].Sealed == row.held; kept != row.kept {
-			t.Errorf("%s: composed with %s (held %s), kept %v, want %v", row.name, rows[0].Sealed, row.held, kept, row.kept)
+		for k, v := range c.placements {
+			again.placements[k] = v
 		}
+		for k, v := range c.records {
+			again.records[k] = v
+		}
+		for k, v := range c.modules {
+			again.modules[k] = v
+		}
+		c.mu.Unlock()
+		var composed string
+		for _, u := range again.compose(ctx) {
+			if u.Key == keeperKey {
+				composed = string(u.Value)
+			}
+		}
+		if composed == "" {
+			t.Fatalf("%s: keeper was not composed", row.name)
+		}
+		if _, box := boxOf(composed); (box == row.held) != row.kept {
+			t.Errorf("%s: composed with %s (held %s), kept %v", row.name, box, row.held, row.kept)
+		}
+	}
+	// And the node's own key (review of L2, round 3, finding 1): a core box its own composition carries is held there, and
+	// kept after a restart as a module's is, so the node's own composition does not move either.
+	ownComp := delivery.Composed{Node: "n1", Resources: []json.RawMessage{json.RawMessage(`{"id":"core-box","sealed":"{\"ask\":999}"}`)}}
+	b.mu.Lock()
+	sealedAt[`{"ask":999}`] = generation + 1
+	b.mu.Unlock()
+	coreEnv := env
+	coreEnv.Unseal = func(box string) ([]byte, error) {
+		b.mu.Lock()
+		defer b.mu.Unlock()
+		if g, ok := sealedAt[box]; ok {
+			return []byte(fmt.Sprintf("password-%d", g)), nil
+		}
+		return nil, fmt.Errorf("not sealed to n1: %s", box)
+	}
+	coreEnv.Held = func(module string) (delivery.Seals, []string) {
+		if module == agent.NodeModule {
+			return delivery.Seals{"core-box#sealed": rotatedBox}, nil
+		}
+		return nil, nil
+	}
+	if got := delivery.SealsIn(New(coreEnv).keepRunning(ownComp, nil))["core-box#sealed"]; got != rotatedBox {
+		t.Errorf("the node's own core box after a restart is %s, want the one its key holds %s", got, rotatedBox)
 	}
 
 	// The pair credentials asked (ADR 0550 §5): the keys go to the vault's own with their role, a consumer's comes back as
@@ -460,10 +537,15 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 	if err != nil || len(creds) != 1 || len(issued) != 1 {
 		t.Fatalf("credentials %+v issued %+v, %v", creds, issued, err)
 	}
-	if creds[0].State != compose.CredentialHeld || creds[0].Provider != "n2" || creds[0].Sealed != `{"run":1}` ||
+	if creds[0].State != compose.CredentialHeld || creds[0].Provider != "n2" || creds[0].Sealed != `{"c":1}` ||
 		issued[0].Consumer != "n3" || issued[0].ForProvider != `{"p":1}` {
 		t.Errorf("credentials %+v issued %+v", creds, issued)
 	}
+	// What runs at the credential's slot is kept, after the composition (keepRunning, issue 1126, C1).
+	webComp := delivery.Composed{Node: "n1", Module: "web", Resources: []json.RawMessage{json.RawMessage(`{"id":"web.secret-pg","sealed":"{\"c\":1}"}`)}}
+	if got := delivery.SealsIn(pc.keepRunning(webComp, pairEnv.Running()))["web.secret-pg#sealed"]; got != `{"run":1}` {
+		t.Errorf("web's credential composed as %s, want the box that runs", got)
+	}
 	// A node that cannot compose tells the agent once, and once again when it composes (the agent's haves say it).
 	pc.refused("a stand-in for web's password")
 	pc.refused("a stand-in for web's password")
@@ -718,6 +800,11 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 		Pull: delivery.Head{Repository: delivery.CatalogueRepository, Number: 1, Head: "h-same"}, Placement: &trunk})
 	sameKey, _ := delivery.PlacementCandidateKey("same", "n1")
 	nb.put(delivery.PlacementCandidatesBucket, sameKey, sameRaw, 3)
+	// Issue 1126, T3: an older group (a closed pull request's candidate, rev 2) is answered after the newest.
+	oldRaw, _ := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Group: "older",
+		Pull: delivery.Head{Repository: delivery.CatalogueRepository, Number: 2, Head: "h-old"}, Placement: &trunk})
+	oldKey, _ := delivery.PlacementCandidateKey("older", "n1")
+	nb.put(delivery.PlacementCandidatesBucket, oldKey, oldRaw, 2)
 	for _, bucket := range []string{delivery.PlacementCandidatesBucket, delivery.PlacementBucket, delivery.HaveBucket, delivery.ModulesBucket, ResolvedBucket} {
 		nb.send(bucket, agent.Update{Caught: true})
 	}
@@ -742,11 +829,31 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 		if g := a["same"]; g.Verdict != delivery.CandidateSame || !strings.Contains(g.Note, "this answer took over") {
 			t.Errorf("the shadow, once ready, answered %+v", g)
 		}
+		// The newest first, handed at once: the older group still holds the answer it had.
+		if g := a["older"]; g.Verdict != delivery.CandidateRefused {
+			t.Errorf("the older group was answered before the newest was handed: %+v", g)
+		}
+		if g := (<-nanswered)["older"]; g.Verdict != delivery.CandidateSame {
+			t.Errorf("the older group, answered next: %+v", g)
+		}
+		// Issue 1126, T2: an answer says when it starts and when it ends (and, every minute between, what it waits on).
 		nmu.Lock()
-		if !strings.Contains(strings.Join(nsaid, "\n"), "the answer to same took") {
-			t.Errorf("a slow answer was not said: %q", nsaid)
+		said := strings.Join(nsaid, "\n")
+		for _, line := range []string{"the answer to same took", "answering placement candidate same (revision 3)",
+			"answered placement candidate same in", "answering placement candidate older (revision 2)"} {
+			if !strings.Contains(said, line) {
+				t.Errorf("%q was not said: %q", line, nsaid)
+			}
+		}
+		if strings.Index(said, "answering placement candidate same") > strings.Index(said, "answering placement candidate older") {
+			t.Errorf("the older group was answered first: %q", nsaid)
 		}
 		nmu.Unlock()
+		done := nsh.c.waiting("the vault's own secrets")
+		if got := nsh.c.waitingOn(); got != "the vault's own secrets" {
+			t.Errorf("a running answer would say it waits on %q", got)
+		}
+		done()
 	case <-time.After(10 * time.Second):
 		t.Fatal("the shadow, once ready, answered nothing")
 	}
@@ -830,7 +937,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 	buildKey, _ := delivery.BuilderBuildKey("lemurs", fp)
 	b.mu.Lock()
 	b.kv[delivery.BuilderBuildsBucket+"/"+buildKey] = built
-	b.kv[ResolvedBucket+"/n2.mailer.domain"] = []byte(`{"value":"example.org"}`)
+	b.kv[ResolvedBucket+"/n2.mailer.domain"] = []byte(`{"value":"example.org","source":"module"}`)
 	b.reads = nil
 	b.mu.Unlock()
 	fetched := 0

internal/selfcompose/shadow.go

@@ -276,26 +276,92 @@ func (s *Shadow) notReady(ctx context.Context, ins []input, caught []bool) {
 	}
 }
 
-// answerAll answers every group and hands the answers when any changed.
+// answerAll answers every group, the newest candidate first, and hands the answers each time one changes (issue 1126,
+// T3: ace answered a closed pull request's group for 22 minutes before the newer one, all handed at the end). Newest
+// is the highest revision of the group's keys: a closed pull request's candidate is the older and waits; the indexer
+// drops a closed one's keys, which ends its answer here (a node does not read the forge). Each answer says when it
+// starts, every progressEvery while it runs what it waits on, and when it ends (T2: the node said nothing for 22
+// minutes).
 func (s *Shadow) answerAll(ctx context.Context) {
 	c := s.c
 	node := c.env.Node
 	s.mu.Lock()
-	groups := make([]string, 0, len(s.candidates))
-	for g := range s.candidates {
-		groups = append(groups, g)
+	type listed struct {
+		group string
+		rev   uint64
+	}
+	var groups []listed
+	for g, keys := range s.candidates {
+		l := listed{group: g}
+		for _, k := range keys {
+			l.rev = max(l.rev, k.rev)
+		}
+		groups = append(groups, l)
+	}
+	// Each group's last answer stands until it is answered again; a group whose keys went is dropped.
+	next := map[string]delivery.CandidateAnswer{}
+	for _, l := range groups {
+		if a, ok := s.answers[l.group]; ok {
+			next[l.group] = a
+		}
 	}
 	s.mu.Unlock()
-	sort.Strings(groups)
-	next := map[string]delivery.CandidateAnswer{}
-	for _, group := range groups {
+	sort.Slice(groups, func(i, j int) bool {
+		if groups[i].rev != groups[j].rev {
+			return groups[i].rev > groups[j].rev
+		}
+		return groups[i].group < groups[j].group
+	})
+	hand := func() {
+		s.mu.Lock()
+		same := len(next) == len(s.answers)
+		for g, a := range next {
+			if old, ok := s.answers[g]; !ok || !sameAnswer(old, a) {
+				same = false
+			}
+		}
+		out := make(map[string]delivery.CandidateAnswer, len(next))
+		for g, a := range next {
+			out[g] = a
+		}
+		if !same {
+			s.answers = out
+		}
+		s.mu.Unlock()
+		if !same && s.answer != nil {
+			given := make(map[string]delivery.CandidateAnswer, len(out))
+			for g, a := range out {
+				given[g] = a
+			}
+			s.answer(ctx, given)
+			c.env.Say(fmt.Sprintf("%s answered %d placement candidate group(s)", node, len(given)))
+		}
+	}
+	for _, l := range groups {
+		if ctx.Err() != nil {
+			return
+		}
+		group := l.group
 		began := time.Now()
-		slow := time.AfterFunc(slowAfter, func() {
-			c.env.Say(fmt.Sprintf("the answer to %s has run over %s, and goes on", group, slowAfter))
-		})
+		c.env.Say(fmt.Sprintf("answering placement candidate %s (revision %d)", group, l.rev))
+		stop := make(chan struct{})
+		go func() {
+			tick := time.NewTicker(progressEvery)
+			defer tick.Stop()
+			for {
+				select {
+				case <-stop:
+					return
+				case <-tick.C:
+					c.env.Say(fmt.Sprintf("the answer to %s has run %s and waits on %s", group,
+						time.Since(began).Round(time.Second), c.waitingOn()))
+				}
+			}
+		}()
 		a, ok := s.answerGroup(ctx, group)
-		slow.Stop()
-		if took := time.Since(began); took > slowAfter {
+		close(stop)
+		took := time.Since(began)
+		if took > slowAfter {
 			c.env.Say(fmt.Sprintf("the answer to %s took %s", group, took.Round(time.Second)))
 			if ok {
 				// Words that do not move with the time it took, so an answer the same as before is not written again.
@@ -303,35 +369,27 @@ func (s *Shadow) answerAll(ctx context.Context) {
 			}
 		}
 		if ok {
+			a = delivery.FitAnswer(a, delivery.MaxAnswerBytes)
 			next[group] = a
+			c.env.Say(fmt.Sprintf("answered placement candidate %s in %s: %s", group, took.Round(time.Millisecond), a.Verdict))
+		} else {
+			delete(next, group)
+			c.env.Say(fmt.Sprintf("placement candidate %s does not place this node: nothing to answer", group))
 		}
+		hand()
 	}
-	s.mu.Lock()
-	same := len(next) == len(s.answers)
-	for g, a := range next {
-		if old, ok := s.answers[g]; !ok || !sameAnswer(old, a) {
-			same = false
-		}
-	}
-	if !same {
-		s.answers = next
-	}
-	s.mu.Unlock()
-	if !same && s.answer != nil {
-		out := make(map[string]delivery.CandidateAnswer, len(next))
-		for g, a := range next {
-			out[g] = a
-		}
-		s.answer(ctx, out)
-		c.env.Say(fmt.Sprintf("%s answered %d placement candidate group(s)", node, len(out)))
-	}
+	hand() // a group whose keys went, when no group is left to answer
 }
 
+// How often a running answer says what it waits on.
+var progressEvery = time.Minute
+
 // answerGroup is the node's answer to one group, and false when the node has nothing to answer: neither the trunk nor
 // the group places it.
 func (s *Shadow) answerGroup(ctx context.Context, group string) (delivery.CandidateAnswer, bool) {
 	c := s.c
 	node := c.env.Node
+	defer c.waiting("composing")()
 	s.mu.Lock()
 	keys := s.candidates[group]
 	given := make(map[string]*delivery.Placement, len(keys))