Node shadow: reuse a running seal only at its own slot, lay values as the controller does, answer the newest candidate first and say how it goes (issue 1126, C1 C2, T1 T2 T3)

A box is reused by slot (resource, field) after the composition, from what runs or the
have's slots; a have written before slots is reused within its module alone. Config
values are laid by mesh-sdk's one rule (never a default). The shadow answers the
newest candidate first and hands each answer as it lands, says when an answer starts,
every minute what it waits on, and when it ends, and fits an answer to the node's key.
Needs mesh-sdk fix/1126-seed-same (16b8d8a1).

internal/selfcompose/selfcompose.go

@@ -11,7 +11,6 @@
 package selfcompose
 
 import (
-	"bytes"
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
@@ -56,14 +55,18 @@ type Env struct {
 	// have says it (agent.NotComposed): a refusal is loud where haves are read, not one journal line. Nil tells nobody.
 	Refused func(why string)
 	// Running is what the node runs now (agent.Everything), and Unseal opens a value sealed to this machine: a secret
-	// the vault seals again for each ask is kept as the box the node already runs when that box opens to the same
-	// value, so neither the switch to composing its own nor a restart changes a composition (ADR 0547 §3). Either nil
-	// keeps the vault's box.
+	// the vault seals again for each ask is kept as the box the node already runs at the same slot (the same field of
+	// the same resource of the same module, delivery.KeepSeals) when that box opens to the same value, so neither the
+	// switch to composing its own nor a restart changes a composition (ADR 0547 §3); a box held at another slot is
+	// never taken, though it opens to the same value (issue 1126, C1: a provider's grant and its consumer's secret).
+	// Either nil keeps the vault's box.
 	Running func() []delivery.Composed
 	Unseal  func(sealed string) ([]byte, error)
-	// Held is the sealed values a module's have says it runs (delivery.Have.Sealed), kept by the agent on the bus: what
-	// runs after a restart, before the agent applied anything, and after the switch once the kept declaration is old.
-	Held func(module string) []string
+	// Held is the sealed values a module's have says it runs, kept by the agent on the bus: what runs after a restart,
+	// before the agent applied anything, and after the switch once the kept declaration is old. By slot
+	// (delivery.Have.SealedAt); legacy is the list a have written before slots holds (delivery.Have.Sealed), reused
+	// only within its own module.
+	Held func(module string) (slots delivery.Seals, legacy []string)
 	// Fetch reads an archive at a URL in the artifact store: a moved module's templates archive, whose texts its built
 	// manifest carries (issue 1126). Read once per digest. Nil reads none, and a moved module does not compose.
 	Fetch func(ctx context.Context, url string) ([]byte, error)
@@ -93,8 +96,12 @@ type Composer struct {
 	sealedOwn map[string]ownSealed
 	said      string
 	refusal   string
-	caught    bool
-	rev       uint64
+	// doing is what the composer waits on now (waiting), for a running answer to say (issue 1126, T2); its own lock, so
+	// it is read while c.mu is held.
+	doingMu sync.Mutex
+	doing   string
+	caught  bool
+	rev     uint64
 }
 
 // New is a composer for one node.
@@ -399,11 +406,10 @@ func (c *Composer) takeValue(u agent.Update) {
 	}
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	var r struct {
-		Value json.RawMessage `json:"value"`
-	}
-	var v any
-	if u.Deleted || json.Unmarshal(u.Value, &r) != nil || len(r.Value) == 0 || json.Unmarshal(r.Value, &v) != nil {
+	// Only a value its source lays: a default is the composition's already, as the controller reads them
+	// (catalogue.LaidValue, issue 1126, C2: Element's links blanked by a laid default).
+	v, laid := catalogue.LaidValue(u.Value)
+	if u.Deleted || !laid {
 		delete(c.values[module], key)
 		return
 	}
@@ -479,6 +485,10 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
 	}
 	c.refused("")
 	c.sayOnce("")
+	// The boxes it runs, slot by slot: read before c.mu is taken, since the agent's lock is taken to read what runs.
+	for key, comp := range compositions {
+		compositions[key] = c.keepRunning(comp, running)
+	}
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	var out []agent.Update
@@ -532,45 +542,69 @@ func (c *Composer) refused(why string) {
 	}
 }
 
-// runningBox is the box the node already runs that opens to the same value as box, or box: what the agent runs, and
-// what each wanted module's have says it runs sealed (Held). Read before c.mu is taken, since the agent's lock is taken
-// to read what runs.
-func (c *Composer) runningBox(box string) string {
-	if c.env.Unseal == nil || (c.env.Running == nil && c.env.Held == nil) {
-		return box
+// keepRunning is comp with each sealed value the node already runs at the same slot kept, where it opens to the same
+// value (delivery.KeepSeals, issue 1126, C1): of what runs (running, the agent's), else of what its module's have says
+// it runs (Held). A have written before slots names no slot: its boxes are reused within their own module alone, at a
+// slot no box is kept for, and only one opening to the slot's value.
+func (c *Composer) keepRunning(comp delivery.Composed, running []delivery.Composed) delivery.Composed {
+	if c.env.Unseal == nil {
+		return comp
 	}
-	value, err := c.env.Unseal(box)
-	if err != nil {
-		return box
-	}
-	var held []string
-	if c.env.Running != nil {
-		for _, comp := range c.env.Running() {
-			held = append(held, agent.SealedIn(comp)...)
-		}
+	module := comp.Module
+	if module == "" {
+		module = agent.NodeModule
 	}
+	kept := delivery.Seals{}
+	var legacy []string
 	if c.env.Held != nil {
-		c.mu.Lock()
-		// The node's own key among them (agent.NodeModule): its core boxes, as its last run said them.
-		modules := make([]string, 0, len(c.wants))
-		for m := range c.wants {
-			modules = append(modules, m)
+		slots, old := c.env.Held(module)
+		for slot, box := range slots {
+			kept[slot] = box
 		}
-		c.mu.Unlock()
-		sort.Strings(modules)
-		for _, m := range modules {
-			held = append(held, c.env.Held(m)...)
+		legacy = old
+	}
+	for _, r := range running {
+		if r.Module == comp.Module {
+			for slot, box := range delivery.SealsIn(r) {
+				kept[slot] = box
+			}
 		}
 	}
-	for _, h := range held {
-		if h == box {
+	for slot, box := range delivery.SealsIn(comp) {
+		if _, ok := kept[slot]; ok {
 			continue
 		}
-		if v, err := c.env.Unseal(h); err == nil && bytes.Equal(v, value) {
-			return h
+		for _, h := range legacy {
+			if same, known := c.sameValue(box, h); known && same {
+				kept[slot] = h
+				break
+			}
 		}
 	}
-	return box
+	return delivery.KeepSeals(comp, kept, c.sameValue)
+}
+
+// waiting notes what the composer waits on until the returned func is called, which restores what it waited on before.
+func (c *Composer) waiting(what string) func() {
+	c.doingMu.Lock()
+	before := c.doing
+	c.doing = what
+	c.doingMu.Unlock()
+	return func() {
+		c.doingMu.Lock()
+		c.doing = before
+		c.doingMu.Unlock()
+	}
+}
+
+// waitingOn is what the composer waits on now, in words.
+func (c *Composer) waitingOn() string {
+	c.doingMu.Lock()
+	defer c.doingMu.Unlock()
+	if c.doing == "" {
+		return "nothing it reads (composing)"
+	}
+	return c.doing
 }
 
 func (c *Composer) sayOnce(line string) {
@@ -652,6 +686,7 @@ func (r reader) Manifest(ctx context.Context, digest string) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
+	defer r.c.waiting("the manifest " + digest + " (" + delivery.ManifestsBucket + ")")()
 	value, err := r.c.env.Get(ctx, delivery.ManifestsBucket, key)
 	if err != nil {
 		return nil, err
@@ -674,6 +709,7 @@ func (r reader) Build(ctx context.Context, module, fp string) (*delivery.Builder
 	if err != nil {
 		return nil, err
 	}
+	defer r.c.waiting("the build of " + module + " at " + fp + " (" + delivery.BuilderBuildsBucket + ")")()
 	raw, err := r.c.env.Get(ctx, delivery.BuilderBuildsBucket, key)
 	if err != nil || raw == nil {
 		return nil, err
@@ -700,6 +736,7 @@ func (r reader) Archive(ctx context.Context, url, digest string) ([]byte, error)
 		return nil, fmt.Errorf("this node reads no archive (%s)", url)
 	}
 	// Bounded: a store that does not answer must not hold the composer or the node's answer (issue 1126).
+	defer r.c.waiting("the archive " + url)()
 	fetching, cancel := context.WithTimeout(ctx, archiveWithin)
 	defer cancel()
 	body, err := r.c.env.Fetch(fetching, url)
@@ -719,17 +756,15 @@ func (r reader) Archive(ctx context.Context, url, digest string) ([]byte, error)
 // Served reads, of another node's module, the keys named of mesh-config's resolved values, one key each: the node is
 // granted those keys alone (buscompose Principal.ServedReads, issue 1126).
 func (r reader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) {
+	defer r.c.waiting("what " + module + " serves on " + node + " (" + ResolvedBucket + ")")()
 	out := map[string]any{}
 	for _, k := range keys {
 		raw, err := r.c.env.Get(ctx, ResolvedBucket, node+"."+module+"."+k)
 		if err != nil {
 			return nil, err
 		}
-		var v struct {
-			Value json.RawMessage `json:"value"`
-		}
-		var value any
-		if raw == nil || json.Unmarshal(raw, &v) != nil || len(v.Value) == 0 || json.Unmarshal(v.Value, &value) != nil {
+		value, laid := catalogue.LaidValue(raw)
+		if raw == nil || !laid {
 			continue
 		}
 		out[k] = value
@@ -760,7 +795,9 @@ func (r reader) OwnSecrets(ctx context.Context, node string) ([]compose.OwnSecre
 	if node != r.c.env.Node || r.c.env.Own == nil {
 		return nil, nil
 	}
+	done := r.c.waiting("the vault's own secrets")
 	raw, err := r.c.env.Own(ctx, map[string]any{})
+	done()
 	if err != nil {
 		return nil, fmt.Errorf("the vault's own: %w", err)
 	}
@@ -781,25 +818,16 @@ func (r reader) OwnSecrets(ctx context.Context, node string) ([]compose.OwnSecre
 	if a.Node != node {
 		return nil, fmt.Errorf("the vault answered %q's own secrets to %s", a.Node, node)
 	}
-	ids := make([]string, len(a.Own))
-	fresh := make([]string, len(a.Own))
-	for i, o := range a.Own {
-		ids[i] = o.Module + "/" + o.Name
-		if !r.c.keeps(ids[i], o.Generation, a.Key) {
-			fresh[i] = r.c.runningBox(string(o.Sealed))
-		}
-	}
 	r.c.mu.Lock()
 	defer r.c.mu.Unlock()
 	out := make([]compose.OwnSecretRow, 0, len(a.Own))
-	for i, o := range a.Own {
-		id := ids[i]
+	for _, o := range a.Own {
+		// The box first had is kept while its generation stands, so sealing it again moves no composition; the box the
+		// node runs at the same slot is kept after the composition (keepRunning).
+		id := o.Module + "/" + o.Name
 		kept, ok := r.c.sealedOwn[id]
 		if !ok || kept.generation != o.Generation || kept.key != a.Key {
-			kept = ownSealed{generation: o.Generation, key: a.Key, sealed: fresh[i]}
-			if kept.sealed == "" {
-				kept.sealed = string(o.Sealed)
-			}
+			kept = ownSealed{generation: o.Generation, key: a.Key, sealed: string(o.Sealed)}
 			r.c.sealedOwn[id] = kept
 		}
 		out = append(out, compose.OwnSecretRow{Node: node, Module: o.Module, Name: o.Name, Origin: o.Origin,
@@ -823,7 +851,9 @@ func (r reader) Credentials(ctx context.Context, node string, asConsumer, asProv
 	for _, k := range asProvider {
 		ask = append(ask, asked{Role: "provider", CredentialKey: k})
 	}
+	done := r.c.waiting("the vault's credentials")
 	raw, err := r.c.env.Own(ctx, map[string]any{"credentials": ask})
+	done()
 	if err != nil {
 		return nil, nil, fmt.Errorf("the vault's own: %w", err)
 	}
@@ -844,28 +874,17 @@ func (r reader) Credentials(ctx context.Context, node string, asConsumer, asProv
 	if a.Node != node {
 		return nil, nil, fmt.Errorf("the vault answered %q's credentials to %s", a.Node, node)
 	}
-	ids := make([]string, len(a.Credentials))
-	fresh := make([]string, len(a.Credentials))
-	for i, c := range a.Credentials {
-		ids[i] = c.Role + "|" + c.Name + "|" + c.Provision + "|" + c.Consumer + "|" + c.ConsumerModule + "|" + c.Local
-		if !r.c.keeps(ids[i], c.Generation, a.Key) {
-			fresh[i] = r.c.runningBox(string(c.Sealed))
-		}
-	}
 	r.c.mu.Lock()
 	defer r.c.mu.Unlock()
 	var creds []compose.Credential
 	var issued []compose.Issued
-	for i, c := range a.Credentials {
-		// The value first had is kept while its generation stands, so sealing it again moves no composition; and the
-		// first had is the box the node runs when it opens to the same value (runningBox).
-		id := ids[i]
+	for _, c := range a.Credentials {
+		// The value first had is kept while its generation stands, so sealing it again moves no composition; the box the
+		// node runs at the same slot is kept after the composition (keepRunning).
+		id := c.Role + "|" + c.Name + "|" + c.Provision + "|" + c.Consumer + "|" + c.ConsumerModule + "|" + c.Local
 		kept, ok := r.c.sealedOwn[id]
 		if !ok || kept.generation != c.Generation || kept.key != a.Key {
-			kept = ownSealed{generation: c.Generation, key: a.Key, sealed: fresh[i]}
-			if kept.sealed == "" {
-				kept.sealed = string(c.Sealed)
-			}
+			kept = ownSealed{generation: c.Generation, key: a.Key, sealed: string(c.Sealed)}
 			r.c.sealedOwn[id] = kept
 		}
 		k := c.CredentialKey
@@ -881,14 +900,6 @@ func (r reader) Credentials(ctx context.Context, node string, asConsumer, asProv
 	return creds, issued, nil
 }
 
-// keeps says the composer holds id's sealed value at this generation and key already.
-func (c *Composer) keeps(id string, generation int, key string) bool {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	kept, ok := c.sealedOwn[id]
-	return ok && kept.generation == generation && kept.key == key
-}
-
 // Placed reads whether the catalogue places node now, for the node agent choosing whose compositions it applies at
 // each start: true when its placement key holds a value, false only when a read succeeds and finds no key or a delete
 // marker. A read that fails chooses neither (review of the shadow, round 1: a timeout or a bus not ready at a restart