Review round 1 of the bus want: a removal keeps the bus server, the have says a server made anew runs the want's fingerprint, the old build is read before the first write, and an unanswered inspect fails the apply rather than replacing the server (ADR 0594 §2, §4)

cmd/mesh-host/nodeagent.go

@@ -123,8 +123,8 @@ func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
 		Sealed: x.Sealed, Server: x.Server})
 }
 
-func (h sessionHaves) Removed(ctx context.Context, module string) {
-	h.session.NoteRemoved(ctx, module, "")
+func (h sessionHaves) Removed(ctx context.Context, module string, words ...string) {
+	h.session.NoteRemoved(ctx, module, "", words...)
 }
 
 // shorten keeps a have's detail within what a have value holds: its head, which says what failed, and its tail, which is
@@ -564,9 +564,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
 				sayFacts(ctx, ag)
 				return res
 			},
-			Remove: func(ctx context.Context, module string, others []delivery.Composed) agent.Result {
+			Remove: func(ctx context.Context, module string, others []delivery.Composed, keep map[string]bool) agent.Result {
 				actx, done := applyContext(session.WithLink(ctx))
-				res := removeComposition(actx, opts, module, others, ag.Placed(module), sched, say)
+				res := removeComposition(actx, opts, module, others, ag.Placed(module), keep, sched, say)
 				done()
 				afterApply()
 				sayFacts(ctx, ag)
@@ -732,13 +732,13 @@ func applyComposition(ctx context.Context, opts options, c delivery.Composed, ot
 	if got.Probe != nil {
 		return failedWith(got.Probe.partial().Error())
 	}
-	return agent.Result{}
+	return agent.Result{Kept: got.Outcome.Kept}
 }
 
 // removeComposition removes one module's own resources and nothing of another's.
-func removeComposition(ctx context.Context, opts options, module string, others []delivery.Composed, placed map[string]bool,
+func removeComposition(ctx context.Context, opts options, module string, others []delivery.Composed, placed, keep map[string]bool,
 	sched *apply.Scheduler, say link.Announce) agent.Result {
-	scope := apply.Scope{Module: module, Others: resourcesOf(others), Placed: placed}
+	scope := apply.Scope{Module: module, Others: resourcesOf(others), Placed: placed, Keep: keep}
 	got := applyModuleHeld(ctx, opts, &declaration.Declaration{Version: declaration.Version}, &scope, sched, say)
 	switch {
 	case got.Refused != "":
@@ -746,7 +746,7 @@ func removeComposition(ctx context.Context, opts options, module string, others
 	case got.Err != nil:
 		return agent.Result{Failed: true, Detail: shorten(got.Err.Error())}
 	}
-	return agent.Result{}
+	return agent.Result{Kept: got.Outcome.Kept}
 }
 
 // busyModules are the modules whose last apply left a container running because it said busy.

internal/agent/agent.go

@@ -77,8 +77,9 @@ type Haves interface {
 	Read(ctx context.Context, module string) (Have, bool)
 	// Write writes a module's have where it differs from the bucket's.
 	Write(ctx context.Context, h Have)
-	// Removed writes the module's have with the phase removed, once, if a key of it exists.
-	Removed(ctx context.Context, module string)
+	// Removed writes the module's have with the phase removed, once, if a key of it exists; words, when given, are its
+	// detail.
+	Removed(ctx context.Context, module string, words ...string)
 }
 
 // Result is how one apply or removal came out.
@@ -88,6 +89,8 @@ type Result struct {
 	// JetStream API error while the bus reloads), not the module's: the apply is tried again without a new revision.
 	Transient bool
 	Detail    string
+	// Kept is the resources of the keep set the apply left as they ran, by id (apply.Report.Kept, novox/hq ADR 0594 §4).
+	Kept []string
 }
 
 // Backup is the answer of the backup asked before a module moves.
@@ -117,8 +120,9 @@ type Env struct {
 	// resources (catalogue.BusServerOf, novox/hq ADR 0594 §7), empty for any other module; known says whether the
 	// module's manifest could be read. Nil names no bus module.
 	BusServer func(ctx context.Context, module string) (ids []string, known bool)
-	// Remove removes the module's own resources and nothing of another's.
-	Remove func(ctx context.Context, module string, others []delivery.Composed) Result
+	// Remove removes the module's own resources and nothing of another's, and leaves those of keep as they run (the bus
+	// server's, ADR 0594 §2).
+	Remove func(ctx context.Context, module string, others []delivery.Composed, keep map[string]bool) Result
 	// Backup backs a module up before a new composition of it is applied; nil backs up nothing.
 	Backup func(ctx context.Context, module string) Backup
 	// Kept is the resources the node last applied, by module, from the declaration it kept before the switch (ADR 0539
@@ -612,6 +616,10 @@ func (a *Agent) writeHave(ctx context.Context, h Have) {
 
 // serverOf is what the module's have says the bus server runs (Have.Server): kept since this agent read or wrote it,
 // else read from the have once. Empty for every module but the bus.
+//
+// A have written before this node agent kept the server names none: the server then runs what that have says the module
+// runs, its fingerprint, else its old-form build (the take-over of a server the node's own composition made). Read so on
+// the first read, before this agent writes the have, so every have it writes carries it.
 func (a *Agent) serverOf(ctx context.Context, module string) string {
 	a.mu.Lock()
 	s, known := a.server[module]
@@ -619,8 +627,17 @@ func (a *Agent) serverOf(ctx context.Context, module string) string {
 	if known {
 		return s
 	}
+	if server, _ := a.busKeep(ctx, module); len(server) == 0 {
+		return "" // not the bus, or not known to be yet: nothing kept, so a later read asks again
+	}
 	if h, ok := a.env.Haves.Read(ctx, module); ok {
 		s = h.Server
+		if s == "" {
+			s = h.FP
+		}
+		if s == "" {
+			s = h.Build
+		}
 	}
 	a.mu.Lock()
 	if _, written := a.server[module]; !written {
@@ -1496,14 +1513,6 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
 	ranServer, madeServer := "", ""
 	if len(keep) > 0 {
 		ranServer = a.serverOf(ctx, module)
-		if ranServer == "" {
-			if h, ok := a.env.Haves.Read(ctx, module); ok {
-				ranServer = h.FP
-				if ranServer == "" {
-					ranServer = h.Build
-				}
-			}
-		}
 		if ranServer == "" {
 			ranServer = fp // a bus module this node never ran: its server is made at this fingerprint
 		}
@@ -1606,6 +1615,10 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
 	if a.env.Settled != nil {
 		a.env.Settled(module)
 	}
+	if len(keep) > 0 && len(res.Kept) == 0 {
+		// Nothing of the server was there to keep: the apply made it at this fingerprint (ADR 0594 §4).
+		madeServer = fp
+	}
 	a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseRunning), Detail: res.Detail,
 		Server: madeServer})
 }
@@ -1624,7 +1637,16 @@ func (a *Agent) remove(ctx context.Context, module string, w wantNow) {
 	}
 	others := a.heldOut(module)
 	a.mu.Unlock()
-	res := a.env.Remove(ctx, module, others)
+	// A removed want of the bus module never removes its server (novox/hq ADR 0594 §2): a removal carries no warrant, and
+	// removing the server is not built. One whose manifest cannot be read keeps what would be the server's, said.
+	keep, known := a.busKeep(ctx, module)
+	if !known {
+		keep = map[string]bool{}
+		for _, id := range catalogue.BusServer {
+			keep[module+"."+id] = true
+		}
+	}
+	res := a.env.Remove(ctx, module, others, keep)
 	a.mu.Lock()
 	if res.Failed {
 		a.attempt[module] = attempt{FP: "removed", WantRev: w.Rev}
@@ -1644,6 +1666,16 @@ func (a *Agent) remove(ctx context.Context, module string, w wantNow) {
 	if a.env.Settled != nil {
 		a.env.Settled(module)
 	}
+	if len(res.Kept) > 0 {
+		words := fmt.Sprintf("removed, but for %s: the bus server stays as it runs until a want carrying the operator's "+
+			"warrant removes it (novox/hq ADR 0594 §2)", strings.Join(res.Kept, ", "))
+		if !known {
+			words += "; its manifest could not be read, so what would be its server was left"
+		}
+		a.env.Say(module + " is " + words)
+		a.env.Haves.Removed(ctx, module, words)
+		return
+	}
 	a.env.Haves.Removed(ctx, module)
 }
 

internal/agent/agent_test.go

@@ -65,10 +65,11 @@ func (f *feed) put(key string, value []byte) {
 }
 
 type haves struct {
-	mu      sync.Mutex
-	stored  map[string]Have
-	log     []Have
-	removed []string
+	mu           sync.Mutex
+	stored       map[string]Have
+	log          []Have
+	removed      []string
+	removedWords []string
 	// unreadable is each module whose have the bus cannot read now (ReadErr).
 	unreadable map[string]bool
 }
@@ -99,10 +100,11 @@ func (h *haves) Write(_ context.Context, x Have) {
 	h.stored[x.Module] = x
 	h.log = append(h.log, x)
 }
-func (h *haves) Removed(_ context.Context, m string) {
+func (h *haves) Removed(_ context.Context, m string, words ...string) {
 	h.mu.Lock()
 	defer h.mu.Unlock()
 	h.removed = append(h.removed, m)
+	h.removedWords = append(h.removedWords, strings.Join(words, ""))
 }
 func (h *haves) last(module string) Have {
 	h.mu.Lock()
@@ -127,7 +129,9 @@ type bed struct {
 	kept      map[string][]json.RawMessage
 	others    [][]delivery.Composed
 	keeps     []map[string]bool // what each apply was told to keep as it runs (ADR 0594 §2)
-	said      []string
+	// serverGone is a bus server not on the machine: an apply keeping it keeps nothing and makes it.
+	serverGone bool
+	said       []string
 	// settled is each module Settled was told, and settledRan whether what runs held it then.
 	settled    []string
 	settledRan []bool
@@ -143,6 +147,13 @@ func newBed(t *testing.T) *bed {
 			defer b.mu.Unlock()
 			b.others = append(b.others, others)
 			b.keeps = append(b.keeps, keep)
+			var kept []string // what of keep was there to keep: all of it, unless the test took the server away
+			for id := range keep {
+				if !b.serverGone {
+					kept = append(kept, id)
+				}
+			}
+			slices.Sort(kept)
 			name := c.Module
 			if name == "" {
 				name = NodeModule
@@ -151,13 +162,19 @@ func newBed(t *testing.T) *bed {
 			if b.failWith != "" && (b.failTimes == 0 || len(b.applied) <= b.failTimes) {
 				return Result{Failed: true, Transient: b.transient, Detail: b.failWith}
 			}
-			return Result{}
+			return Result{Kept: kept}
 		},
-		Remove: func(_ context.Context, m string, _ []delivery.Composed) Result {
+		Remove: func(_ context.Context, m string, _ []delivery.Composed, keep map[string]bool) Result {
 			b.mu.Lock()
 			defer b.mu.Unlock()
 			b.removed = append(b.removed, m)
-			return Result{}
+			b.keeps = append(b.keeps, keep)
+			var kept []string
+			for id := range keep {
+				kept = append(kept, id)
+			}
+			slices.Sort(kept)
+			return Result{Kept: kept}
 		},
 		Backup: func(_ context.Context, m string) Backup {
 			b.mu.Lock()
@@ -1582,8 +1599,22 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
 			return b.keeps[len(b.keeps)-1]
 		}
 		fpA, fpB := fpOf("a"), fpOf("b")
+		// A have written before the first apply (the node cannot compose yet) keeps the old build's name as what the
+		// server runs: the writer names no build, so it is read before the first write.
+		want("nats", fpA, "")
+		eventually(t, "the bus's want was not taken", func() bool {
+			b.ag.mu.Lock()
+			defer b.ag.mu.Unlock()
+			return b.ag.wants["nats"].Present
+		})
+		b.ag.NotComposed(context.Background(), "a stand-in for the bus's secret")
+		if h := b.haves.last("nats"); h.Build != "" || h.Server != took {
+			t.Fatalf("a have written before the first apply: %+v", h)
+		}
+		b.ag.NotComposed(context.Background(), "")
 		for i, row := range []struct {
 			name, fp, image, users, warrant string
+			gone                            bool // the server is not on the machine: made, at the want's fingerprint
 			applies                         bool
 			keep                            map[string]bool
 			server                          string
@@ -1595,7 +1626,12 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
 				applies: true, server: fpB},
 			{name: "the same warrant again", fp: fpB, image: "2", users: "u2", warrant: delivery.WarrantOperator, server: fpB},
 			{name: "a build after the warrant", fp: fpA, image: "3", users: "u2", applies: true, keep: server, server: fpB},
+			{name: "a build whose server is not there", fp: fpOf("e"), image: "4", users: "u2", gone: true, applies: true,
+				keep: server, server: fpOf("e")},
 		} {
+			b.mu.Lock()
+			b.serverGone = row.gone
+			b.mu.Unlock()
 			before := len(b.appliedList())
 			key, raw, dig := bus(row.fp, row.image, row.users)
 			b.comps.put(key, raw)
@@ -1645,9 +1681,28 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
 			}
 		}
 		time.Sleep(100 * time.Millisecond)
-		if got := b.appliedList(); len(got) != 7 {
+		if got := b.appliedList(); len(got) != 8 {
 			t.Errorf("applied %v, not once for each change", got)
 		}
+		// A removed want removes the rest of the bus and never its server: it is kept, said in the journal and in the
+		// removed have's words.
+		rkey, _ := delivery.WantKey(node, "nats")
+		rraw, _ := delivery.EncodeWant(delivery.Want{Removed: true})
+		b.wants.put(rkey, rraw)
+		eventually(t, "the bus's removal was not asked", func() bool {
+			b.haves.mu.Lock()
+			defer b.haves.mu.Unlock()
+			return slices.Contains(b.haves.removed, "nats")
+		})
+		if got := lastKeep(); !maps.Equal(got, server) {
+			t.Errorf("the bus's removal kept %v, not its server %v", got, server)
+		}
+		b.haves.mu.Lock()
+		words := b.haves.removedWords[len(b.haves.removedWords)-1]
+		b.haves.mu.Unlock()
+		if !strings.Contains(words, "the bus server stays") || !strings.Contains(words, "nats.server") {
+			t.Errorf("the removed have does not say the server stays: %q", words)
+		}
 	})
 }
 

internal/apply/apply.go

@@ -120,6 +120,9 @@ type Report struct {
 	MovedOut []MovedOut `json:"moved-out,omitempty"`
 	// Left is every socket, pipe or device a move-out directory holds and did not move (ADR 0422 §4).
 	Left []LeftInPlace `json:"left,omitempty"`
+	// Kept is every resource the scope keeps (Scope.Keep) that this apply left as it runs, by id (novox/hq ADR 0594
+	// §2, §4): what says whether the bus server still runs what it ran, or was made by this apply.
+	Kept []string `json:"kept,omitempty"`
 }
 
 // Waiting says whether a container of this apply waits to be replaced because it is busy: what makes the
@@ -416,6 +419,11 @@ func applyScoped(
 	if err := refuseRootIdentities(d); err != nil {
 		return report, known, &Error{Resource: "the declaration's identities", Err: err, Done: report}
 	}
+	// What of the bus server is kept as it runs, asked of the runtime before anything moves (ADR 0594 §2).
+	keepNow, err := keptServer(ctx, d, scope, run)
+	if err != nil {
+		return report, known, &Error{Resource: "the bus server", Err: err, Done: report}
+	}
 
 	declared := map[string]bool{}
 	for _, r := range d.Resources {
@@ -544,6 +552,17 @@ func applyScoped(
 	removeOrphan := func(orphan store.Applied) error {
 		var action, detail string
 		var err error
+		if keepNow[orphan.ID] {
+			// A removed want of the bus module removes the rest of it and never its server (ADR 0594 §2): only a
+			// warranted want replaces or removes the server. Its record stays, the module's.
+			detail := "no longer declared; the bus server is left as it runs: only a want carrying the operator's " +
+				"warrant removes it (novox/hq ADR 0594 §2)"
+			report.Outcomes = append(report.Outcomes, Outcome{ID: orphan.ID, Type: orphan.Type, Target: orphan.Target,
+				Action: keptAsItRuns, Detail: detail})
+			report.Kept = append(report.Kept, orphan.ID)
+			log(fmt.Sprintf("  %s %s (%s): %s", keptAsItRuns, orphan.ID, orphan.Target, detail))
+			return nil
+		}
 		if module, held := heldByOthers[orphan.ID]; held && !declared[orphan.ID] {
 			if module == "" {
 				module = "another module"
@@ -1071,8 +1090,9 @@ func applyScoped(
 			if from, handed := handedFiles[resource.Identity()]; handed && was.ID == "" {
 				previous.Wrote = from.Wrote
 			}
-			if o, kept := keepAsItRuns(ctx, scope, resource, run, previous); kept {
+			if o, kept := keepAsItRuns(keepNow, resource, previous); kept {
 				outcome = o
+				report.Kept = append(report.Kept, o.ID)
 			} else {
 				outcome, err = applyOne(ctx, sys, resource, run, changed, in, previous, unseal, keepFound)
 			}
@@ -3041,8 +3061,8 @@ const keptAsItRuns = "kept-as-it-runs"
 // unpacked at its path before. The outcome carries what the resource was last recorded reading and holding, so its
 // record is the module's from here on (taken over from the node's own composition at the bus module's first want) and
 // a later warranted apply still sees what changed under it.
-func keepAsItRuns(ctx context.Context, scope *Scope, r declaration.Resource, run Runner, previous store.Applied) (Outcome, bool) {
-	if scope == nil || !scope.Keep[r.Identity()] {
+func keepAsItRuns(keep map[string]bool, r declaration.Resource, previous store.Applied) (Outcome, bool) {
+	if !keep[r.Identity()] {
 		return Outcome{}, false
 	}
 	out := begin(r)
@@ -3053,9 +3073,7 @@ func keepAsItRuns(ctx context.Context, scope *Scope, r declaration.Resource, run
 		if res.Schedule != "" || res.RunOnce {
 			return Outcome{}, false
 		}
-		if _, err := containerState(ctx, res.Name, run); err != nil {
-			return Outcome{}, false
-		}
+		// Found running by keptServer before the loop: it is left whatever its declaration says now.
 		out.reads = previous.Reads
 		out.busySince = previous.BusySince
 		out.Detail = why
@@ -3079,6 +3097,36 @@ func keepAsItRuns(ctx context.Context, scope *Scope, r declaration.Resource, run
 	return Outcome{}, false
 }
 
+// keptServer is what of the scope's Keep this apply keeps (novox/hq ADR 0594 §2), decided once before anything is
+// applied, as the archives that restart the server come before it: every kept container the declaration names is
+// asked of the runtime once. All found: the Keep stands. One the runtime says is not there: nothing is kept, so the
+// server is made at what the declaration says now and its archives are applied with it. A runtime that does not
+// answer, or answers anything else, is an error, never taken as an absent server: replacing a running bus server
+// because one inspect failed is the harm the Keep exists to prevent.
+func keptServer(ctx context.Context, d *declaration.Declaration, scope *Scope, run Runner) (map[string]bool, error) {
+	if scope == nil || len(scope.Keep) == 0 {
+		return nil, nil
+	}
+	for _, r := range d.Resources {
+		c, ok := r.(*declaration.Container)
+		if !ok || !scope.Keep[r.Identity()] || c.Schedule != "" || c.RunOnce {
+			continue
+		}
+		cri, err := containerRuntime(ctx, run)
+		if err != nil {
+			return nil, fmt.Errorf("%w, so whether the bus server %s runs cannot be told; nothing was applied", err, c.Name)
+		}
+		if _, err := run(ctx, cri, "container", "inspect", "--format", "{{.State.Running}}", c.Name); err != nil {
+			if absent(err) {
+				return nil, nil // made, at what the declaration says now
+			}
+			return nil, fmt.Errorf("the container runtime did not say whether the bus server %s runs (%v); nothing was "+
+				"applied, so it is not replaced on an unanswered look", c.Name, err)
+		}
+	}
+	return scope.Keep, nil
+}
+
 // heldStill is the action of a container left alone because a maintenance window holds it stopped
 // (novox/hq issue 224). Its own word rather than "unchanged": a person reading a machine that looks
 // half-stopped needs to see why, and "unchanged" beside a stopped server says the opposite.

internal/apply/module_scope_test.go

@@ -2,9 +2,10 @@ package apply
 
 import (
 	"context"
-	"fmt"
+	"errors"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"testing"
 
@@ -225,8 +226,24 @@ func TestAResourceThatMovedToItsModulesCompositionIsLeftByTheNodesOwnApply(t *te
 	// while the bus was folded. Kept (a want without the warrant's mark), the server's container and its program archive
 	// are left as they run, whatever the build changed in them, and become nats's; its files and the tools' account file
 	// are written. Not kept, the same apply replaces the container.
-	for _, keep := range []bool{true, false} {
-		t.Run(fmt.Sprintf("the bus server kept %v", keep), func(t *testing.T) {
+	// Review of 774, round 1: the keep is decided once before anything moves, by the runtime's answer for the server. A
+	// server not there is made and its archives applied with it, none said kept; a runtime that does not answer fails the
+	// apply, nothing written and nothing replaced; a removal of the bus removes the rest and leaves the server.
+	for _, tc := range []struct {
+		name   string
+		keep   bool
+		gone   bool // the server is not on the machine
+		mute   bool // the runtime does not answer the server's inspect
+		remove bool // the bus module's want is removed
+	}{
+		{name: "kept", keep: true},
+		{name: "not kept"},
+		{name: "kept, the server not there", keep: true, gone: true},
+		{name: "kept, the runtime does not answer", keep: true, mute: true},
+		{name: "kept, the bus removed", keep: true, remove: true},
+	} {
+		t.Run("the bus server "+tc.name, func(t *testing.T) {
+			keep := tc.keep && !tc.gone
 			dir := t.TempDir()
 			conf, account, program := filepath.Join(dir, "nats.conf"), filepath.Join(dir, "tools", "account"), filepath.Join(dir, "program")
 			bus := func(env, confBody, programBody, module string) *declaration.Declaration {
@@ -250,29 +267,54 @@ func TestAResourceThatMovedToItsModulesCompositionIsLeftByTheNodesOwnApply(t *te
 			if made == nil {
 				t.Fatalf("the node's own made no server: %v", m.asked)
 			}
+			if tc.gone {
+				delete(m.containers, "mesh-broker-nats")
+			}
 			m.asked = nil
+			run := m.run
+			if tc.mute {
+				run = func(ctx context.Context, name string, args ...string) (string, error) {
+					if name == "docker" && len(args) > 1 && args[0] == "container" && args[1] == "inspect" {
+						m.asked = append(m.asked, "docker "+strings.Join(args, " "))
+						return "", errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock")
+					}
+					return m.run(ctx, name, args...)
+				}
+			}
 			next := bus("2", "b", "two", "nats")
 			next.Resources = append(next.Resources, declare(t, `{"id":"nats.tools-account","type":"file","path":"`+account+
 				`","content":"tools","module":"nats"}`).Resources...)
 			scope := Scope{Module: "nats", Placed: map[string]bool{"nats": true}}
-			if keep {
+			if tc.keep {
 				scope.Keep = map[string]bool{"nats.server": true, "nats.program-snapshot": true}
 			}
+			moved := func() []string {
+				var out []string
+				for _, a := range m.asked {
+					for _, verb := range []string{"docker rm", "docker run", "docker create", "docker stop", "docker restart", "docker rename"} {
+						if strings.HasPrefix(a, verb+" ") {
+							out = append(out, a)
+						}
+					}
+				}
+				return out
+			}
 			report, after, err := ApplyModule(context.Background(), archHost(t), next, scope, state,
-				store.OriginDeclared, m.run, nil, nil, nil, nil)
+				store.OriginDeclared, run, nil, nil, nil, nil)
+			if tc.mute {
+				if err == nil || !strings.Contains(err.Error(), "did not say whether the bus server") {
+					t.Fatalf("an unanswered inspect did not fail the apply: %v", err)
+				}
+				if got, _ := os.ReadFile(conf); string(got) != "a" || len(moved()) > 0 {
+					t.Errorf("an apply that could not tell whether the server runs wrote %q and moved %v", got, moved())
+				}
+				return
+			}
 			if err != nil {
 				t.Fatal(err)
 			}
-			var moved []string
-			for _, a := range m.asked {
-				for _, verb := range []string{"docker rm", "docker run", "docker create", "docker stop", "docker restart"} {
-					if strings.HasPrefix(a, verb+" ") {
-						moved = append(moved, a)
-					}
-				}
-			}
-			if keep == (len(moved) > 0) {
-				t.Fatalf("kept %v, and the server was moved by %v", keep, moved)
+			if keep == (len(moved()) > 0) {
+				t.Fatalf("kept %v, and the server was moved by %v", keep, moved())
 			}
 			if got, _ := os.ReadFile(conf); string(got) != "b" {
 				t.Errorf("the server's file holds %q", got)
@@ -283,7 +325,8 @@ func TestAResourceThatMovedToItsModulesCompositionIsLeftByTheNodesOwnApply(t *te
 			if got, _ := os.ReadFile(filepath.Join(program, "snapshot")); keep != (string(got) == "one") {
 				t.Errorf("kept %v, and the server's program is %q", keep, got)
 			}
-			for _, id := range []string{"nats.server", "nats.program-snapshot"} {
+			var kept []string
+			for _, id := range []string{"nats.program-snapshot", "nats.server"} {
 				rec, ok := after.Find(id)
 				if !ok || rec.Module != "nats" {
 					t.Errorf("%s is recorded %+v, not as nats's", id, rec)
@@ -291,10 +334,35 @@ func TestAResourceThatMovedToItsModulesCompositionIsLeftByTheNodesOwnApply(t *te
 				if o := outcomeOf(report, id); keep != (o.Action == keptAsItRuns) {
 					t.Errorf("kept %v, and %s was %s: %s", keep, id, o.Action, o.Detail)
 				}
+				if keep {
+					kept = append(kept, id)
+				}
 			}
-			if keep && report.Changed() == false {
+			if !slices.Equal(report.Kept, kept) && !(len(kept) == 0 && len(report.Kept) == 0) {
+				t.Errorf("the report says it kept %v, want %v", report.Kept, kept)
+			}
+			if tc.keep && report.Changed() == false {
 				t.Error("an apply that wrote the server's files says nothing changed")
 			}
+			if !tc.remove {
+				return
+			}
+			m.asked = nil
+			gone, last, err := ApplyModule(context.Background(), archHost(t), &declaration.Declaration{}, scope, after,
+				store.OriginDeclared, run, nil, nil, nil, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(moved()) > 0 || m.containers["mesh-broker-nats"] == nil || !exists(filepath.Join(program, "snapshot")) {
+				t.Errorf("the bus's removal removed its server: %v", moved())
+			}
+			if exists(conf) || exists(account) {
+				t.Errorf("the bus's removal left the rest of it: conf %v, account %v", exists(conf), exists(account))
+			}
+			if _, ok := last.Find("nats.server"); !ok || !slices.Equal(gone.Kept, []string{"nats.program-snapshot", "nats.server"}) &&
+				!slices.Equal(gone.Kept, []string{"nats.server", "nats.program-snapshot"}) {
+				t.Errorf("the removal kept %v, the server recorded %v", gone.Kept, ok)
+			}
 		})
 	}
 }

internal/link/have.go

@@ -59,6 +59,8 @@ type haveState struct {
 	// nodeKeeps is what the node agent keeps as it runs on a want without the warrant's mark (HaveNode.Keeps, novox/hq
 	// ADR 0594 §8), written into its key.
 	nodeKeeps []string
+	// removedWords is, by module in removed, what its removed have says when it is more than that it was removed.
+	removedWords map[string]string
 	// written is the build of each module this node-engine wrote a have key for and that the declaration it applied
 	// still holds; removed is each such module the latest declaration no longer holds, with the build its key last
 	// said, until the key is written once with the phase removed (issue 1369, novox/hq ADR 0408 §4: never deleted).
@@ -508,6 +510,9 @@ func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string) b
 		}
 		h := delivery.Have{V: delivery.Version, Build: s.removed[module], Phase: delivery.PhaseRemoved,
 			Health: delivery.HealthUnknown, Detail: "the applied declaration no longer holds the module"}
+		if words := s.removedWords[module]; words != "" {
+			h.Detail = words
+		}
 		raw, err := delivery.EncodeHave(h)
 		if err != nil {
 			q.sayHave(fmt.Sprintf("cannot encode %s: %v", key, err))
@@ -699,13 +704,23 @@ func (q *Session) NoteUserList(ctx context.Context, module string, w delivery.Us
 
 // NoteRemoved takes that a module's own resources were removed, and writes its have with the phase removed once (issue
 // 1369): the key is never deleted.
-func (q *Session) NoteRemoved(ctx context.Context, module, build string) {
+//
+// words, when given, are the removed have's detail (the bus server left as it runs, novox/hq ADR 0594 §2).
+func (q *Session) NoteRemoved(ctx context.Context, module, build string, words ...string) {
 	q.init()
 	s := &q.haves
 	s.mu.Lock()
 	if s.removed == nil {
 		s.removed = map[string]string{}
 	}
+	if said := strings.Join(words, ""); said != "" {
+		if s.removedWords == nil {
+			s.removedWords = map[string]string{}
+		}
+		s.removedWords[module] = said
+	} else {
+		delete(s.removedWords, module)
+	}
 	if f, ok := s.modules[module]; ok && build == "" {
 		build = f.Build
 	}

internal/link/have_test.go

@@ -586,6 +586,12 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth(
 	if got := l.have(t, "n1.nats"); got.FP != fp || got.Server != "build-1792674595610176744" {
 		t.Fatalf("the bus's have: %+v", got)
 	}
+	// Its removal leaves the server, and its removed have says so (review of 774, round 1).
+	q.NoteRemoved(context.Background(), "nats", "", "removed, but for nats.server: the bus server stays as it runs")
+	if got := l.have(t, "n1.nats"); got.Phase != delivery.PhaseRemoved || !strings.Contains(got.Detail, "the bus server stays") {
+		t.Fatalf("the bus's removed have: %+v", got)
+	}
+	q.NoteModule(context.Background(), ModuleFact{Module: "nats", FP: fp, Composed: dig, Server: "build-1792674595610176744"})
 	q.NoteComposes(context.Background(), delivery.ComposesOwn, "it composes its own", delivery.KeepsBusServer)
 	if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || !slices.Equal(n.Keeps, []string{delivery.KeepsBusServer}) ||
 		n.Composes != delivery.ComposesOwn {