Merge pull request 'Node agent: a failed placement read picks no owner, a deleted placement returns the node to the controller without a restart (issue 1126, group fix/1126-shadow-all)' (#154) from fix/1126-shadow-all into main

cmd/mesh-host/nodeagent.go

@@ -544,9 +544,21 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
 				session.NoteCandidates(ctx, answers)
 			})
 		go shadow.Keep(linked, link.Backoff)
-		if composer := selfComposer(linked, session, node, say, hooks); composer != nil {
+		composer, why := selfComposer(linked, session, node, say, hooks)
+		if linked.Err() != nil {
+			// The link ended before the node knew whose compositions to apply: it chose neither, and waits for the next.
+			stopRun()
+			cancel()
+			continue
+		}
+		if composer != nil {
 			env.Composed, env.SelfComposed, env.Trusted, env.Policy = composer, true, nil, composer.Policy
+			session.NoteComposes(linked, delivery.ComposesOwn, why)
+			// And once the catalogue no longer places it, the agent begins again on the controller's compositions: the
+			// placement key deleted ends this run, the way back (issue 1126, the seed's revert).
+			go endWhenUnplaced(run, session, node, stopRun, say)
 		} else {
+			session.NoteComposes(linked, delivery.ComposesController, why)
 			// And once the catalogue places it, the agent begins again composing its own: the placement key appearing ends
 			// this run, so a node does not wait for a restart to follow the seed of its placement (ADR 0549 §4).
 			go endWhenPlaced(run, session, node, stopRun, say)
@@ -843,23 +855,25 @@ type composerHooks struct {
 	held    func(module string) []string
 }
 
-func selfComposer(ctx context.Context, session *link.Session, node string, say func(string), hooks composerHooks) *selfcompose.Composer {
+func selfComposer(ctx context.Context, session *link.Session, node string, say func(string), hooks composerHooks) (*selfcompose.Composer, string) {
 	key, err := delivery.PlacementKey(node)
 	if err != nil {
 		say("this node applies the controller's compositions: " + err.Error())
-		return nil
+		return nil, err.Error()
 	}
-	got, err := session.KVHistory(ctx, delivery.PlacementBucket, key)
-	if err != nil || len(got) == 0 || got[len(got)-1].Deleted {
-		why := "the catalogue does not place it yet"
-		if err != nil {
-			why = "its placement cannot be read: " + err.Error()
-		}
+	env := composerEnv(session, node, say, hooks)
+	placed, err := selfcompose.Placed(ctx, env.Get, node, link.Backoff, say)
+	if err != nil {
+		return nil, "its placement could not be read before the link ended: " + err.Error()
+	}
+	if !placed {
+		why := "the catalogue does not place it"
 		say("this node applies the controller's compositions: " + why)
-		return nil
+		return nil, why
 	}
-	say("this node composes its own modules: the catalogue places it (" + key + ")")
-	return selfcompose.New(composerEnv(session, node, say, hooks))
+	why := "the catalogue places it (" + key + ")"
+	say("this node composes its own modules: " + why)
+	return selfcompose.New(env), why
 }
 
 // composerEnv is what the node's composers read of the bus, the vault and the agent: its own composer and its answer
@@ -889,6 +903,38 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com
 		Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held}
 }
 
+// endWhenUnplaced calls stop once this node's placement key is deleted from the indexer's placement (after the watch
+// caught up, or gone already), so the agent begins again and applies the controller's compositions, as a node the
+// catalogue does not place does from its start: the way back (issue 1126, the seed's revert). A watch that cannot be
+// made is said; the node then returns at its next start.
+func endWhenUnplaced(ctx context.Context, session *link.Session, node string, stop func(), say link.Announce) {
+	key, err := delivery.PlacementKey(node)
+	if err != nil {
+		return
+	}
+	updates, err := bucketWatch{session, delivery.PlacementBucket}.Watch(ctx, key)
+	if err != nil {
+		say("whether the catalogue still places this node is not followed (" + err.Error() + "): it returns to the controller's compositions at its next start")
+		return
+	}
+	placed := false
+	for u := range updates {
+		switch {
+		case u.Caught && placed:
+			continue
+		case u.Caught || u.Deleted || len(u.Value) == 0:
+			say("the catalogue no longer places this node (" + key + "): the node agent begins again, applying the controller's compositions")
+			stop()
+			return
+		default:
+			placed = true
+		}
+	}
+	if ctx.Err() == nil {
+		say("the watch of " + key + " ended while the link stays: whether the catalogue still places this node is followed again at the node agent's next start")
+	}
+}
+
 // endWhenPlaced calls stop once this node's placement key appears in the indexer's placement (a value after the watch
 // caught up, or one already there), so the agent begins again and composes its own. A watch that cannot be made is said;
 // the node then switches at its next start.
@@ -909,4 +955,7 @@ func endWhenPlaced(ctx context.Context, session *link.Session, node string, stop
 			return
 		}
 	}
+	if ctx.Err() == nil {
+		say("the watch of " + key + " ended while the link stays: whether the catalogue places this node is followed again at the node agent's next start")
+	}
 }

cmd/mesh-host/nodeagent_integration_test.go

@@ -402,8 +402,56 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
 	placement, resolved, candidates := st.placement, st.resolved, st.candidates
 	hkey, place, fp := st.hkey, st.place, st.fp
 	within(t, 5*time.Second, "the node did not compose and apply its own module once placed", func() bool { return b.file("web") == "greeting=bonjour\n" })
-	_ = placement
 	h8rest(t, b, resolved, candidates, hkey, place, fp)
+	h8back(t, b, placement, candidates, hkey)
+}
+
+// h8back is the way back (issue 1126, the seed's revert): a candidate deleting the node's placement is compared with
+// the controller's compositions at its wants (here none: refused, naming web), and the placement key deleted returns the agent to them without a
+// restart, said in the journal and the node's have, keeping what runs.
+func h8back(t *testing.T, b *itBed, placement, candidates nats.KeyValue, hkey string) {
+	t.Helper()
+	node := func() delivery.HaveNode {
+		e, err := b.haves.Get(hkey)
+		if err != nil {
+			return delivery.HaveNode{}
+		}
+		n, _ := delivery.DecodeHaveNode(e.Value())
+		return n
+	}
+	if n := node(); n.Composes != delivery.ComposesOwn {
+		t.Errorf("a node composing its own says it composes %q", n.Composes)
+	}
+	craw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Group: "revert/x",
+		Pull: delivery.Head{Repository: delivery.CatalogueRepository, Number: 2, Head: "h2"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ckey, _ := delivery.PlacementCandidateKey("revert/x", itNode)
+	if _, err := candidates.Put(ckey, craw); err != nil {
+		t.Fatal(err)
+	}
+	// The controller holds no composition at web's want (the node composed its own): returning would leave web with
+	// nothing to apply, and the answer says so rather than "same" (review of the shadow, round 1).
+	within(t, 10*time.Second, "the node did not refuse returning to compositions the controller does not hold", func() bool {
+		a := node().Candidates["revert/x"]
+		return a.Head == "h2" && a.Returns == "" && a.Verdict == delivery.CandidateRefused &&
+			strings.Contains(a.Refused, "web: the controller holds no composition")
+	})
+	ran := b.have("web").Composed
+	pkey, _ := delivery.PlacementKey(itNode)
+	if err := placement.Delete(pkey); err != nil {
+		t.Fatal(err)
+	}
+	within(t, 15*time.Second, "the node did not return to the controller's compositions when its placement went", func() bool {
+		b.mu.Lock()
+		said := strings.Join(b.said, "\n")
+		b.mu.Unlock()
+		return strings.Contains(said, "the catalogue no longer places this node") && node().Composes == delivery.ComposesController
+	})
+	if h := b.have("web"); h.Composed != ran || h.Phase == delivery.PhaseFailed || b.file("web") != "greeting=salut\n" {
+		t.Errorf("returning to the controller's compositions moved what runs: %+v, %q", h, b.file("web"))
+	}
 }
 
 // h8state is what h8 set up that the rest of H8 writes to.

go.mod

@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027201213-7a8e9f970648
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027223231-e32d77753024
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0
 	golang.org/x/crypto v0.57.0

go.sum

@@ -18,6 +18,12 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027200317-538ffbaa085b h1:LuBK5dtB
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027200317-538ffbaa085b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027201213-7a8e9f970648 h1:XoMsgg9kSgjQWa86NZM8h8OpxJ2kovxcMS2C0i6WSVU=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027201213-7a8e9f970648/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027221705-2fd5ffdd6263 h1:JZV8C+WR6NPiopj9ShzhbpjN0gqVQV60vOFxsxFwH00=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027221705-2fd5ffdd6263/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027222254-472d5f314466 h1:xzbkI9dOkN/Iv78AS/fOwhjkgBEQOYtpbHzNYwb4eu0=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027222254-472d5f314466/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027223231-e32d77753024 h1:MnmyspVoaCldYmm4Ut1O39RDR9WpiL4TfFgDEnY+QRg=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027223231-e32d77753024/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=

internal/link/have.go

@@ -52,6 +52,9 @@ type haveState struct {
 	nodeSealed []string
 	// nodeCandidates is the node's answer to each placement candidate group (novox/hq ADR 0559 §4), written into its key.
 	nodeCandidates map[string]delivery.CandidateAnswer
+	// nodeComposes and nodeComposesWhy are whose compositions the node applies and why (HaveNode.Composes), written into
+	// its key.
+	nodeComposes, nodeComposesWhy string
 	// written is the build of each module this node-engine wrote a have key for and that the declaration it applied
 	// still holds; removed is each such module the latest declaration no longer holds, with the build its key last
 	// said, until the key is written once with the phase removed (issue 1369, novox/hq ADR 0408 §4: never deleted).
@@ -423,6 +426,7 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
 			facts.Composed = s.nodeComposed
 			facts.Sealed = s.nodeSealed
 			facts.Candidates = s.nodeCandidates
+			facts.Composes, facts.ComposesWhy = s.nodeComposes, s.nodeComposesWhy
 			if raw, err = delivery.EncodeHaveNode(facts); err == nil {
 				want := facts
 				if !q.putHave(write, bucket, key, raw, func(current []byte) bool {
@@ -752,6 +756,26 @@ func (q *Session) NoteCandidates(ctx context.Context, answers map[string]deliver
 	q.writeHaves(ctx, bus)
 }
 
+// NoteComposes says, in the node's own key, whose compositions the node applies (delivery.ComposesOwn or
+// delivery.ComposesController) and why: written at each start of its agent, so a node returning to the controller's
+// compositions says so where every have is read (issue 1126).
+func (q *Session) NoteComposes(ctx context.Context, composes, why string) {
+	q.init()
+	s := &q.haves
+	s.mu.Lock()
+	changed := s.nodeComposes != composes || s.nodeComposesWhy != why
+	s.nodeComposes, s.nodeComposesWhy = composes, why
+	known := s.node != nil
+	s.mu.Unlock()
+	if !known || !changed {
+		return // said with the node's first facts
+	}
+	q.mu.Lock()
+	bus := q.bus
+	q.mu.Unlock()
+	q.writeHaves(ctx, bus)
+}
+
 // ReadNodeHave is the node's own key as the bucket holds it, and whether it holds one: what the node's own composition
 // it last ran is and carries sealed, read after a restart (ADR 0547 §3).
 func (q *Session) ReadNodeHave(ctx context.Context) (delivery.HaveNode, bool) {

internal/selfcompose/selfcompose.go

@@ -16,16 +16,20 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"sort"
 	"strings"
 	"sync"
+	"time"
 
 	"git.novox.be/novox/mesh-sdk/go/catalogue"
 	"git.novox.be/novox/mesh-sdk/go/compose"
 	"git.novox.be/novox/mesh-sdk/go/compose/frombus"
 	"git.novox.be/novox/mesh-sdk/go/delivery"
 
+	"github.com/nats-io/nats.go"
+
 	"github.com/novox/mesh-host/internal/agent"
 )
 
@@ -428,15 +432,27 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
 	_, nodeWanted := c.wants[agent.NodeModule]
 	// What it runs inside its own composition and no want names at a build of the new form stays there, as the
 	// controller's composed writer folds it (issue 1126); compose folds none a want names.
-	inOwn := c.inOwnOf(c.placements[c.env.Node])
+	own, placed := c.placements[c.env.Node]
+	inOwn := c.inOwnOf(own)
 	r := reader{c: c}
 	c.mu.Unlock()
+	if !placed {
+		// The catalogue no longer places it: the agent returns to the controller's compositions (endWhenUnplaced), and
+		// this is no refusal of what it runs (issue 1126, the seed's revert).
+		c.sayOnce("the catalogue no longer places this node: it composes nothing of its own, and keeps running what it runs")
+		return nil
+	}
 	// Composed from the build its want names (frombus.Scope.KeptAt, issue 1126).
 	for _, module := range behind {
 		kept[module] = fps[module]
 	}
 
-	compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn}, fps)
+	// What it runs: a folded module of an old-form build is carried in its own composition as it runs (issue 1126).
+	var running []delivery.Composed
+	if c.env.Running != nil {
+		running = c.env.Running()
+	}
+	compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running}, fps)
 	if err != nil {
 		c.refused(fmt.Sprintf("%v", err))
 		c.sayOnce(fmt.Sprintf("the node cannot compose its modules, and keeps running what it runs: %v", err))
@@ -650,6 +666,9 @@ func (r reader) Build(ctx context.Context, module, fp string) (*delivery.Builder
 	return &b, nil
 }
 
+// archiveWithin bounds one read of a templates archive in the artifact store.
+const archiveWithin = 30 * time.Second
+
 // Archive reads a templates archive once per digest (an archive is the whole of its digest), checked against it.
 func (r reader) Archive(ctx context.Context, url, digest string) ([]byte, error) {
 	r.c.mu.Lock()
@@ -661,7 +680,10 @@ func (r reader) Archive(ctx context.Context, url, digest string) ([]byte, error)
 	if r.c.env.Fetch == nil {
 		return nil, fmt.Errorf("this node reads no archive (%s)", url)
 	}
-	body, err := r.c.env.Fetch(ctx, url)
+	// Bounded: a store that does not answer must not hold the composer or the node's answer (issue 1126).
+	fetching, cancel := context.WithTimeout(ctx, archiveWithin)
+	defer cancel()
+	body, err := r.c.env.Fetch(fetching, url)
 	if err != nil {
 		return nil, err
 	}
@@ -847,3 +869,38 @@ func (c *Composer) keeps(id string, generation int, key string) bool {
 	kept, ok := c.sealedOwn[id]
 	return ok && kept.generation == generation && kept.key == key
 }
+
+// Placed reads whether the catalogue places node now, for the node agent choosing whose compositions it applies at
+// each start: true when its placement key holds a value, false only when a read succeeds and finds no key or a delete
+// marker. A read that fails chooses neither (review of the shadow, round 1: a timeout or a bus not ready at a restart
+// flipped a self-composing node to the controller's compositions for a whole run): it is tried again after wait(try),
+// each distinct failure said once, until ctx ends, which is then its error.
+func Placed(ctx context.Context, get func(ctx context.Context, bucket, key string) ([]byte, error), node string,
+	wait func(try int) time.Duration, say func(string)) (bool, error) {
+	key, err := delivery.PlacementKey(node)
+	if err != nil {
+		return false, err
+	}
+	said := ""
+	for try := 1; ; try++ {
+		raw, err := get(ctx, delivery.PlacementBucket, key)
+		if errors.Is(err, nats.ErrBucketNotFound) {
+			return false, nil // the indexer made no placement state yet: nothing places it, read and known
+		}
+		if err == nil {
+			return len(raw) > 0, nil
+		}
+		if ctx.Err() != nil {
+			return false, ctx.Err()
+		}
+		if why := err.Error(); why != said {
+			say(fmt.Sprintf("whether the catalogue places this node cannot be read (%s): the node agent waits, choosing neither its own compositions nor the controller's: %v", key, err))
+			said = why
+		}
+		select {
+		case <-ctx.Done():
+			return false, ctx.Err()
+		case <-time.After(wait(try)):
+		}
+	}
+}

internal/selfcompose/selfcompose_test.go

@@ -16,6 +16,7 @@ import (
 	"git.novox.be/novox/mesh-sdk/go/compose"
 	"git.novox.be/novox/mesh-sdk/go/compose/frombus"
 	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"github.com/nats-io/nats.go"
 
 	"github.com/novox/mesh-host/internal/agent"
 )
@@ -440,6 +441,18 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 		}
 		return out
 	}
+	// The controller's compositions at n1's wants, in n1's composition bucket: what a head deleting its placement
+	// returns it to (the seed's revert); here the ones it runs.
+	composedBucket, _ := delivery.ComposedBucketFor("n1")
+	for key, c := range running {
+		raw, err := delivery.EncodeComposed(c)
+		if err != nil {
+			t.Fatal(err)
+		}
+		readOnly.mu.Lock()
+		readOnly.kv[composedBucket+"/"+key] = raw
+		readOnly.mu.Unlock()
+	}
 	answered := make(chan map[string]delivery.CandidateAnswer, 8)
 	sh := NewShadow(senv, func(_ context.Context, a map[string]delivery.CandidateAnswer) { answered <- a })
 	sctx, scancel := context.WithCancel(ctx)
@@ -450,7 +463,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 	go sh.Keep(sctx, func(int) time.Duration { return 5 * time.Millisecond })
 	time.Sleep(30 * time.Millisecond)
 	sb.missing(delivery.PlacementCandidatesBucket, false)
-	sb.made(6)
+	sb.made(7)
 	rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}})
 	tp, _ := delivery.EncodePlacement(trunk)
 	sb.put(delivery.PlacementBucket, "placement.n1", tp, 1)
@@ -459,6 +472,21 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 		raw, _ := delivery.EncodeModule(m)
 		sb.put(delivery.ModulesBucket, name, raw, 1)
 	}
+	wantLemurs, _ := delivery.WantKey("n1", "lemurs")
+	var lemursDigest string
+	for _, c := range running {
+		if c.Module == "lemurs" {
+			lemursDigest, _ = delivery.ComposedDigestOf(c)
+		}
+	}
+	wantRaw, err := delivery.EncodeWant(delivery.Want{FP: fp, Composed: lemursDigest})
+	if err != nil {
+		t.Fatal(err)
+	}
+	sb.put(delivery.AssignerWantsBucket, wantLemurs, wantRaw, 1)
+	wantNode, _ := delivery.WantNodeKey("n1")
+	nodeRaw, _ := delivery.EncodeWant(delivery.Want{})
+	sb.put(delivery.AssignerWantsBucket, wantNode, nodeRaw, 1)
 	candidate := func(group string, rev uint64, modules ...string) {
 		p := trunk
 		p.Modules = nil
@@ -477,6 +505,15 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 	candidate("adds", 4, "lemurs", "keeper")
 	candidate("drops", 5)
 	candidate("ghost", 6, "lemurs", "ghost")
+	// A head deleting n1's placement file (the seed's revert, mesh-catalog #526): n1 returns to the controller's
+	// compositions, answered as such, not as every module removed.
+	if raw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Group: "deletes",
+		Pull: delivery.Head{Repository: delivery.CatalogueRepository, Number: 1, Head: "h-deletes"}}); err != nil {
+		t.Fatal(err)
+	} else {
+		k, _ := delivery.PlacementCandidateKey("deletes", "n1")
+		sb.put(delivery.PlacementCandidatesBucket, k, raw, 7)
+	}
 	sb.caught()
 	var answers map[string]delivery.CandidateAnswer
 	waitFor := func(why string, ok func(map[string]delivery.CandidateAnswer) bool) {
@@ -490,7 +527,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 			}
 		}
 	}
-	waitFor("every group answered", func(a map[string]delivery.CandidateAnswer) bool { return len(a) == 4 })
+	waitFor("every group answered", func(a map[string]delivery.CandidateAnswer) bool { return len(a) == 5 })
 	// Every watch and get the shadow makes is one the node's grant allows (issue 1126, 2026-10-27 09:21).
 	sb.mu.Lock()
 	for _, w := range sb.watches {
@@ -520,6 +557,10 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 		"ghost": func(a delivery.CandidateAnswer) bool {
 			return a.Verdict == delivery.CandidateRefused && a.Refused != ""
 		},
+		"deletes": func(a delivery.CandidateAnswer) bool {
+			return a.Verdict == delivery.CandidateSame && a.Returns == delivery.ReturnsToController && a.Revision == 7 &&
+				len(a.Removed) == 0 && fmt.Sprint(a.Same) == "[lemurs node]"
+		},
 	} {
 		if !want(answers[group]) {
 			t.Errorf("%s: %+v", group, answers[group])
@@ -534,7 +575,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 	}
 	k, _ := delivery.PlacementCandidateKey("same", "n1")
 	sb.del(delivery.PlacementCandidatesBucket, k)
-	waitFor("the deleted group's answer stays", func(a map[string]delivery.CandidateAnswer) bool { _, ok := a["same"]; return !ok && len(a) == 3 })
+	waitFor("the deleted group's answer stays", func(a map[string]delivery.CandidateAnswer) bool { _, ok := a["same"]; return !ok && len(a) == 4 })
 
 	// Review of L2b, round 2: a watch ends while the link stays; a group's key is deleted before the shadow watches
 	// again. The new run reads afresh, so that group is no longer answered.
@@ -546,9 +587,84 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
 	sb.missing(delivery.PlacementCandidatesBucket, false)
 	waitFor("a group deleted while the shadow was down is still answered", func(a map[string]delivery.CandidateAnswer) bool {
 		_, ok := a["drops"]
-		return !ok && len(a) == 2
+		return !ok && len(a) == 3
 	})
 
+	// Whose compositions a node applies at a start (review of the shadow, round 1): only a read that succeeds chooses; a
+	// failing read is tried again, said once, and chooses neither, even when the link ends.
+	for name, row := range map[string]struct {
+		answers []error // per try; nil then the value
+		value   []byte
+		end     bool
+		placed  bool
+		tries   int
+		quiet   bool // read and known: nothing to say
+	}{
+		"placed after two failures": {answers: []error{fmt.Errorf("nats: timeout"), fmt.Errorf("nats: timeout")}, value: []byte(`{}`), placed: true, tries: 3},
+		"no key":                    {placed: false, tries: 1},
+		"no placement state yet":    {answers: []error{fmt.Errorf("the bucket %s: %w", delivery.PlacementBucket, nats.ErrBucketNotFound)}, placed: false, tries: 1, quiet: true},
+		"failing until the link ends": {answers: []error{fmt.Errorf("jetstream not ready"), fmt.Errorf("jetstream not ready"),
+			fmt.Errorf("jetstream not ready")}, end: true, tries: 3},
+	} {
+		lctx, lcancel := context.WithCancel(ctx)
+		tries := 0
+		var said []string
+		get := func(context.Context, string, string) ([]byte, error) {
+			tries++
+			if tries <= len(row.answers) {
+				if row.end && tries == len(row.answers) {
+					lcancel()
+				}
+				return nil, row.answers[tries-1]
+			}
+			return row.value, nil
+		}
+		placed, err := Placed(lctx, get, "n1", func(int) time.Duration { return time.Millisecond }, func(s string) { said = append(said, s) })
+		lcancel()
+		if placed != row.placed || tries != row.tries || (err != nil) != row.end || (len(row.answers) > 0 && !row.quiet) != (len(said) == 1) {
+			t.Errorf("%s: placed %v, err %v after %d reads, said %q", name, placed, err, tries, said)
+		}
+	}
+
+	// The way back, row by row (review of the shadow, round 1): the controller's compositions at n1's wants compared
+	// with what runs; one differing is answered so; one the controller holds none of refuses, without returning.
+	var runsList []delivery.Composed
+	for _, c := range running {
+		runsList = append(runsList, c)
+	}
+	lemursKey, _ = delivery.ComposedKey("n1", "lemurs", fp)
+	changed := running[lemursKey]
+	changed.Resources = []json.RawMessage{json.RawMessage(`{"id":"lemurs.service","type":"service","module":"lemurs","unit":"lemurs.service","state":"stopped","boot":"disabled"}`)}
+	changedRaw, _ := delivery.EncodeComposed(changed)
+	for name, row := range map[string]struct {
+		lemurs []byte
+		check  func(delivery.CandidateAnswer) bool
+	}{
+		"the controller's differs": {changedRaw, func(a delivery.CandidateAnswer) bool {
+			return a.Verdict == delivery.CandidateDiffers && a.Returns == delivery.ReturnsToController &&
+				len(a.Differs) == 1 && a.Differs[0].Module == "lemurs" && fmt.Sprint(a.Same) == "[node]"
+		}},
+		"the controller holds none": {nil, func(a delivery.CandidateAnswer) bool {
+			return a.Verdict == delivery.CandidateRefused && a.Returns == "" && strings.Contains(a.Refused, "lemurs: the controller holds no composition")
+		}},
+	} {
+		get := func(_ context.Context, bucket, key string) ([]byte, error) {
+			if key == lemursKey {
+				return row.lemurs, nil
+			}
+			readOnly.mu.Lock()
+			defer readOnly.mu.Unlock()
+			return readOnly.kv[bucket+"/"+key], nil
+		}
+		a := returning(ctx, get, "n1", map[string]delivery.Want{"lemurs": {FP: fp}, agent.NodeModule: {}}, runsList, "h", 1)
+		if !row.check(a) {
+			t.Errorf("%s: %+v", name, a)
+		}
+		if _, err := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": a}}); err != nil {
+			t.Errorf("%s: the answer is not one a have holds: %v", name, err)
+		}
+	}
+
 	// Issue 1126, the seed's review: a module n1 runs inside its own composition (nats, at a build of the old form) is
 	// composed there again, not at the trunk's build as one added; lemurs at the build it runs, keeper (added) at the
 	// trunk's.

internal/selfcompose/shadow.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -31,6 +32,9 @@ type Shadow struct {
 	mu         sync.Mutex
 	candidates map[string]map[string]candidateKey // by group, by node
 	answers    map[string]delivery.CandidateAnswer
+	// wants are the node's wants by module (agent.NodeModule for its own): what a head deleting its placement returns
+	// it to, the controller's compositions at them.
+	wants map[string]delivery.Want
 }
 
 // candidateKey is one candidate key as last read: its value and revision.
@@ -42,7 +46,7 @@ type candidateKey struct {
 // NewShadow is the shadow composer of one node; answer writes the node's answers into its own have key.
 func NewShadow(env Env, answer func(ctx context.Context, answers map[string]delivery.CandidateAnswer)) *Shadow {
 	return &Shadow{c: New(env), answer: answer, candidates: map[string]map[string]candidateKey{},
-		answers: map[string]delivery.CandidateAnswer{}}
+		answers: map[string]delivery.CandidateAnswer{}, wants: map[string]delivery.Want{}}
 }
 
 // Keep runs the shadow until ctx ends, beginning again after backoff(try) whenever it stops: an input it cannot watch
@@ -84,6 +88,7 @@ func (s *Shadow) Run(ctx context.Context) error {
 	defer cancel()
 	s.mu.Lock()
 	s.candidates = map[string]map[string]candidateKey{}
+	s.wants = map[string]delivery.Want{}
 	s.c = New(s.c.env)
 	s.mu.Unlock()
 	c := s.c
@@ -96,6 +101,8 @@ func (s *Shadow) Run(ctx context.Context) error {
 		{ResolvedBucket, node + ".>", (*Composer).takeValue, nil},
 		// What this node runs moved: an apply ends in its module's have.
 		{delivery.HaveBucket, node + ".>", nil, nil},
+		// Its wants: a head deleting its placement returns it to the controller's compositions at them.
+		{delivery.AssignerWantsBucket, "want." + node + ".>", nil, nil},
 	}
 	type tagged struct {
 		in int
@@ -142,6 +149,8 @@ func (s *Shadow) Run(ctx context.Context) error {
 			case t.in == 5:
 				c.takeModuleHave(t.u)
 				dirty = !isNodeKey(t.u.Key) || dirty
+			case t.in == 6:
+				dirty = s.takeWant(t.u) || dirty
 			default:
 				ins[t.in].take(c, t.u)
 				dirty = true
@@ -292,18 +301,16 @@ func (s *Shadow) answerGroup(ctx context.Context, group string) (delivery.Candid
 		running = c.env.Running()
 	}
 	if p, ok := given[node]; ok && p == nil {
-		// The head removes this node's placement file: everything it runs is removed.
-		a := delivery.CandidateAnswer{Head: head, Revision: rev}
-		for _, r := range running {
-			if r.Module == "" {
-				a.Removed = append(a.Removed, delivery.HaveNodeWord)
-			} else {
-				a.Removed = append(a.Removed, r.Module)
-			}
+		// The head deletes this node's placement file: the node then returns to the controller's compositions, as one the
+		// catalogue does not place (endWhenUnplaced). Those at its wants are compared with what runs (issue 1126, the
+		// seed's revert, whose check read "would stop running node").
+		s.mu.Lock()
+		wants := make(map[string]delivery.Want, len(s.wants))
+		for m, w := range s.wants {
+			wants[m] = w
 		}
-		sort.Strings(a.Removed)
-		a.Verdict = delivery.VerdictOf(a)
-		return a, true
+		s.mu.Unlock()
+		return returning(ctx, c.env.Get, node, wants, running, head, rev), true
 	}
 
 	placement := given[node]
@@ -369,3 +376,88 @@ func sameAnswer(a, b delivery.CandidateAnswer) bool {
 	y, errB := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": b}})
 	return errA == nil && errB == nil && string(x) == string(y)
 }
+
+// takeWant takes one of the node's wants, and says whether it changed.
+func (s *Shadow) takeWant(u agent.Update) bool {
+	module := ""
+	var w delivery.Want
+	if _, err := delivery.ParseWantNodeKey(u.Key); err == nil {
+		module = agent.NodeModule
+	} else if node, m, err := delivery.ParseWantKey(u.Key); err == nil && node == s.c.env.Node {
+		module = m
+		if !u.Deleted {
+			if w, err = delivery.DecodeWant(u.Value); err != nil || w.Removed || w.FP == "" {
+				u.Deleted = true
+			}
+		}
+	} else {
+		return false
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	old, had := s.wants[module]
+	if u.Deleted {
+		delete(s.wants, module)
+		return had
+	}
+	s.wants[module] = w
+	return !had || old.FP != w.FP
+}
+
+// returning is the node's answer to a head deleting its placement: it returns to the controller's compositions, each
+// read at the fingerprint its want names in the node's composition bucket and compared with what runs
+// (delivery.CompareComposed). A want the controller holds no composition for, or one that cannot be read, refuses the
+// answer, naming each: returning then would leave that module with nothing to apply.
+func returning(ctx context.Context, get func(ctx context.Context, bucket, key string) ([]byte, error), node string,
+	wants map[string]delivery.Want, running []delivery.Composed, head string, rev uint64) delivery.CandidateAnswer {
+	a := delivery.CandidateAnswer{Head: head, Revision: rev}
+	bucket, err := delivery.ComposedBucketFor(node)
+	if err != nil {
+		a.Refused, a.Verdict = err.Error(), delivery.CandidateRefused
+		return a
+	}
+	modules := make([]string, 0, len(wants))
+	for m := range wants {
+		modules = append(modules, m)
+	}
+	sort.Strings(modules)
+	comps := map[string]delivery.Composed{}
+	var missing []string
+	for _, m := range modules {
+		var key string
+		if m == agent.NodeModule {
+			key, err = delivery.ComposedNodeKey(node)
+		} else {
+			key, err = delivery.ComposedKey(node, m, wants[m].FP)
+		}
+		if err != nil {
+			missing = append(missing, m+": "+err.Error())
+			continue
+		}
+		raw, err := get(ctx, bucket, key)
+		switch {
+		case err != nil:
+			missing = append(missing, fmt.Sprintf("%s: the controller's composition %s cannot be read: %v", m, key, err))
+			continue
+		case raw == nil:
+			missing = append(missing, fmt.Sprintf("%s: the controller holds no composition %s, which its want names", m, key))
+			continue
+		}
+		comp, err := delivery.DecodeComposed(raw)
+		if err != nil {
+			missing = append(missing, fmt.Sprintf("%s: the controller's composition %s: %v", m, key, err))
+			continue
+		}
+		comps[key] = comp
+	}
+	if len(missing) > 0 {
+		a.Refused = "returning to the controller's compositions, " + strings.Join(missing, "; ")
+		a.Verdict = delivery.CandidateRefused
+		return a
+	}
+	cmp := delivery.CompareComposed(comps, running)
+	a.Returns = delivery.ReturnsToController
+	a.Same, a.Differs, a.Added, a.Removed = cmp.Same, cmp.Differs, cmp.Added, cmp.Removed
+	a.Verdict = delivery.VerdictOf(a)
+	return a
+}