Merge pull request 'Apply again a module whose resources are gone, move a module with nothing to back up, and report ok for a running module with nothing to watch (issue 1944)' (#137) from fix/r6-reapply-missing into main

Reviewed-on: #137

cmd/mesh-host/backup_nothing_test.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+// A module that declares nothing to back up on this machine is not backed up before it moves, and so never held for a
+// backup that cannot run (issue 1945, live on 2026-10-25: redis and searxng on ace were held: "redis declares nothing to
+// back up on this machine"). What node-backup.backed-up answers decides it.
+func TestAModuleWithNothingToBackUpIsNotBackedUp(t *testing.T) {
+	for _, c := range []struct {
+		said    string
+		nothing bool
+	}{
+		{`[{"module":"postgres","runs":1,"paths":["/var/lib/postgres"]}]`, true},
+		{`[{"module":"redis","runs":0,"paths":null,"data":[{"item":"cache"}]}]`, true},
+		{`[]`, true},
+		{`[{"module":"redis","runs":1,"paths":["/var/lib/redis"]}]`, false},
+	} {
+		nothing, ok := declaresNothingToBackUp(json.RawMessage(c.said), "redis")
+		if !ok || nothing != c.nothing {
+			t.Errorf("%s: nothing %v (read %v), want %v", c.said, nothing, ok, c.nothing)
+		}
+	}
+	if _, ok := declaresNothingToBackUp(json.RawMessage(`{"error":"x"}`), "redis"); ok {
+		t.Error("an answer that is not the list was read")
+	}
+}

cmd/mesh-host/main.go

@@ -1166,7 +1166,11 @@ func runLink(ctx context.Context, opts options) error {
 		netJudge.set(network.New(network.Machine{Run: apply.ExecRunner, Linked: queue.Linked},
 			hostOf(mine.Membership.Broker)))
 		go judging.Probe(aside)
-		go judgeWhatRuns(aside, judging, queue, say)
+		go judgeWhatRuns(aside, judging, queue, say, func(modules ...string) {
+			if ag := current.get(); ag != nil {
+				ag.Retry(modules...)
+			}
+		})
 	}
 	// The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the
 	// busy containers are asked for again soon (ADR 0360 §11).
@@ -1329,7 +1333,8 @@ const (
 // the bus when a state changed, again every minute while one is not healthy, and every five minutes
 // anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240
 // rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it.
-func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce) {
+func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce, reapply func(modules ...string)) {
+	var missingBefore map[string]bool
 	ticker := time.NewTicker(liveness.LookEvery)
 	defer ticker.Stop()
 	var lastSaid time.Time
@@ -1415,6 +1420,14 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
 					say(unhealthyWords(r))
 				}
 			}
+			// A resource the machine does not have at all is made again: its module's composition applied anew (issue
+			// 1945), once each time the statement changes, never on a clock.
+			var m []string
+			m, missingBefore = newlyMissing(missingBefore, st)
+			if len(m) > 0 && reapply != nil {
+				say(fmt.Sprintf("applying %s again: a resource it declares is not on this machine", strings.Join(m, ", ")))
+				reapply(m...)
+			}
 		}
 		if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
 			unitSt), accountSt), found), accessSt)) {
@@ -2077,3 +2090,35 @@ func untakenModules(d *declaration.Declaration) map[string][]string {
 	}
 	return d.Adoption.Untaken
 }
+
+// missingModules are the modules of the resources a statement says missing, each once, sorted.
+func missingModules(st liveness.Statement) []string {
+	seen := map[string]bool{}
+	var out []string
+	for _, r := range st.Resources {
+		if r.Missing && r.Module != "" && !seen[r.Module] {
+			seen[r.Module] = true
+			out = append(out, r.Module)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// newlyMissing are the modules of the resources a statement says missing that the last statement acted on did not, each
+// once, sorted, and the missing resources now: a resource still missing is not acted on again, so a module whose
+// resource cannot be made is not applied at every change of the statement (the review of issue 1945).
+func newlyMissing(last map[string]bool, st liveness.Statement) ([]string, map[string]bool) {
+	now := map[string]bool{}
+	var fresh liveness.Statement
+	for _, r := range st.Resources {
+		if !r.Missing {
+			continue
+		}
+		now[r.ID] = true
+		if !last[r.ID] {
+			fresh.Resources = append(fresh.Resources, r)
+		}
+	}
+	return missingModules(fresh), now
+}

cmd/mesh-host/nodeagent.go

@@ -105,6 +105,13 @@ func backupNode(session *link.Session) func(ctx context.Context, module string)
 			return agent.Backup{Words: "the end of the backup cannot be heard: " + err.Error()}
 		}
 		defer stop()
+		// A module this machine backs nothing of up moves without a backup: never held for one that cannot run (issue
+		// 1945). An answer that cannot be had or read asks for the backup as before.
+		if said, err := session.AskSeat(ctx, "node-backup", "backed-up", map[string]any{"module": module}, 30*time.Second); err == nil {
+			if nothing, read := declaresNothingToBackUp(said, module); read && nothing {
+				return agent.Backup{Done: true, Words: module + " declares nothing to back up on this machine"}
+			}
+		}
 		asked := time.Now()
 		if _, err := session.AskSeat(ctx, "node-backup", "now", map[string]any{"module": module}, 30*time.Second); err != nil {
 			return agent.Backup{Words: err.Error()}
@@ -286,6 +293,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
 			Composed: bucketWatch{session, composedBucket},
 			Haves:    sessionHaves{session},
 			Kept:     keptByModule(opts, signer),
+			Recorded: recordedIn(opts.state, say),
 			Trusted:  trustedKeys(opts, signer),
 			Backup:   backupNode(session),
 			Say:      say,
@@ -507,3 +515,55 @@ func refuseEmptyWrites(d *declaration.Declaration) error {
 	}
 	return nil
 }
+
+// recordedIn says the engine's applied state holds a record of every resource of a composition: what the agent believes
+// of a have or the kept declaration only when it does, so a resource removed since is made again (issue 1945). A state
+// that cannot be read says yes: the agent then believes what it was told, as before.
+func recordedIn(statePath string, say func(string)) func(ctx context.Context, c delivery.Composed) bool {
+	return func(_ context.Context, c delivery.Composed) bool {
+		known, err := store.Load(statePath)
+		if err != nil {
+			if say != nil {
+				say(fmt.Sprintf("the applied state cannot be read, so what runs is believed as told: %v", err))
+			}
+			return true
+		}
+		filled := make(map[string]bool, len(c.Filled))
+		for _, p := range c.Filled {
+			filled[p] = true
+		}
+		for _, raw := range c.Resources {
+			r, err := declaration.ParseResource(raw)
+			if err != nil {
+				return true
+			}
+			// A file the provisioner client fills is left to it, and has no record until it writes it.
+			if f, ok := r.(*declaration.File); ok && filled[f.Path] {
+				continue
+			}
+			if _, ok := known.Find(r.Identity()); !ok {
+				return false
+			}
+		}
+		return true
+	}
+}
+
+// declaresNothingToBackUp reads node-backup.backed-up's answer for a module: true when this machine backs nothing of it
+// up (no runs, no paths), and whether the answer could be read at all.
+func declaresNothingToBackUp(said json.RawMessage, module string) (nothing bool, read bool) {
+	var list []struct {
+		Module string   `json:"module"`
+		Runs   int      `json:"runs"`
+		Paths  []string `json:"paths"`
+	}
+	if err := json.Unmarshal(said, &list); err != nil {
+		return false, false
+	}
+	for _, m := range list {
+		if m.Module == module && (m.Runs > 0 || len(m.Paths) > 0) {
+			return false, true
+		}
+	}
+	return true, true
+}

cmd/mesh-host/recorded_test.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"path/filepath"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+
+	"github.com/novox/mesh-host/internal/liveness"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// The record check reads the engine's applied state: a composition one of whose resources has no record there is not
+// recorded (issue 1945).
+func TestACompositionIsRecordedOnlyWhenEveryResourceIs(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "state.json")
+	var s store.State
+	s.Record(store.Applied{ID: "ca-trust.trust", Type: "file", Target: "/etc/ca"})
+	if err := store.Save(path, s); err != nil {
+		t.Fatal(err)
+	}
+	c := delivery.Composed{Node: "ace", Module: "ca-trust", Resources: []json.RawMessage{
+		json.RawMessage(`{"id":"ca-trust.trust","type":"file","module":"ca-trust","path":"/etc/ca","mode":"0644","content":"x"}`)}}
+	recorded := recordedIn(path, func(string) {})
+	if !recorded(context.Background(), c) {
+		t.Fatal("a composition whose every resource has a record was not recorded")
+	}
+	c.Resources = append(c.Resources, json.RawMessage(
+		`{"id":"ca-trust.more","type":"file","module":"ca-trust","path":"/etc/ca2","mode":"0644","content":"y"}`))
+	if recorded(context.Background(), c) {
+		t.Fatal("a composition with a resource the state holds no record of was recorded")
+	}
+}
+
+// The modules applied again are those with a missing resource, each once; one down but there is not among them.
+func TestTheModulesAppliedAgainAreThoseWithAMissingResource(t *testing.T) {
+	st := liveness.Statement{Resources: []liveness.State{
+		{Resource: liveness.Resource{Module: "matrix", ID: "matrix.homeserver"}, State: liveness.Unhealthy, Missing: true},
+		{Resource: liveness.Resource{Module: "matrix", ID: "matrix.db"}, State: liveness.Unhealthy, Missing: true},
+		{Resource: liveness.Resource{Module: "ca-trust", ID: "ca-trust.trust"}, State: liveness.Unhealthy, Missing: true},
+		{Resource: liveness.Resource{Module: "web", ID: "web.server"}, State: liveness.Unhealthy},
+	}}
+	if got := missingModules(st); len(got) != 2 || got[0] != "ca-trust" || got[1] != "matrix" {
+		t.Fatalf("applied again: %v", got)
+	}
+}
+
+// A module is applied again only for a resource newly missing: one still missing at the next change of the statement is
+// not acted on again, so a resource that cannot be made is not applied (and backed up) at every unrelated change. One
+// that comes back and goes again is newly missing again (the review of issue 1945).
+func TestOnlyANewlyMissingResourceAppliesItsModuleAgain(t *testing.T) {
+	missing := func(ids ...string) liveness.Statement {
+		var st liveness.Statement
+		for _, id := range ids {
+			st.Resources = append(st.Resources, liveness.State{Resource: liveness.Resource{Module: "matrix", ID: id},
+				State: liveness.Unhealthy, Missing: true})
+		}
+		return st
+	}
+	var last map[string]bool
+	var got []string
+	got, last = newlyMissing(last, missing("matrix.homeserver"))
+	if len(got) != 1 {
+		t.Fatalf("first missing: %v", got)
+	}
+	got, last = newlyMissing(last, missing("matrix.homeserver"))
+	if len(got) != 0 {
+		t.Fatalf("still missing was acted on again: %v", got)
+	}
+	got, last = newlyMissing(last, missing())
+	got, last = newlyMissing(last, missing("matrix.homeserver"))
+	if len(got) != 1 {
+		t.Fatalf("missing again after it came back: %v", got)
+	}
+}
+
+// A file the provisioner client fills is left to it by the apply and has no record until the client writes it: it does
+// not make its composition unrecorded (the review of issue 1945).
+func TestAFilledFileDoesNotMakeItsCompositionUnrecorded(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "state.json")
+	var s store.State
+	s.Record(store.Applied{ID: "ca-trust.trust", Type: "file", Target: "/etc/ca"})
+	if err := store.Save(path, s); err != nil {
+		t.Fatal(err)
+	}
+	c := delivery.Composed{Node: "ace", Module: "ca-trust", Filled: []string{"/etc/ca-filled"}, Resources: []json.RawMessage{
+		json.RawMessage(`{"id":"ca-trust.trust","type":"file","module":"ca-trust","path":"/etc/ca","mode":"0644","content":"x"}`),
+		json.RawMessage(`{"id":"ca-trust.filled","type":"file","module":"ca-trust","path":"/etc/ca-filled","mode":"0644","owner-at-runtime":"owned"}`)}}
+	if !recordedIn(path, func(string) {})(context.Background(), c) {
+		t.Fatal("a filled file the client has not written made its composition unrecorded")
+	}
+}

internal/agent/agent.go

@@ -94,7 +94,10 @@ type Env struct {
 	// composition unsigned, signed by another key, or changed after it was signed is refused and nothing of it is
 	// applied. Nil trusts no key, so nothing is applied.
 	Trusted func() []ed25519.PublicKey
-	Say     func(string)
+	// Recorded says the engine holds a record of every resource of the composition, so it was applied and not removed
+	// since; nil says yes. A composition the kept declaration or a have says runs is applied again when it says no.
+	Recorded func(ctx context.Context, c delivery.Composed) bool
+	Say      func(string)
 }
 
 // Agent follows the wants.
@@ -563,6 +566,29 @@ func (a *Agent) settleOne(ctx context.Context, module string) {
 // own have (what this node-engine last applied), else the declaration it kept before the switch (ADR 0539 §9). True when
 // it is, with the have written; nothing is applied.
 func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp compNow) bool {
+	// What a have or the kept declaration says runs is believed only when the engine still holds a record of each of its
+	// resources: a resource removed since (a node's own apply took it, issue 1945) is made again by an apply.
+	if a.env.Recorded != nil && !a.env.Recorded(ctx, comp.C) {
+		if module != NodeModule {
+			if _, ok := a.env.Haves.Read(ctx, module); ok {
+				a.mu.Lock()
+				a.prior[module] = true
+				a.mu.Unlock()
+			}
+		}
+		if a.env.Kept != nil {
+			if _, ok := a.env.Kept()[module]; ok {
+				a.mu.Lock()
+				a.prior[module] = true
+				a.mu.Unlock()
+			}
+		}
+		a.mu.Lock()
+		a.sayOnce("unrecorded:"+module+":"+comp.Dig, fmt.Sprintf("%s's composition %s is applied again: this engine holds "+
+			"no record of some of its resources", module, short(comp.Dig)))
+		a.mu.Unlock()
+		return false
+	}
 	if module != NodeModule {
 		if h, ok := a.env.Haves.Read(ctx, module); ok {
 			a.mu.Lock()
@@ -718,3 +744,13 @@ func Canonical(raw json.RawMessage) (json.RawMessage, error) {
 	}
 	return json.Marshal(v)
 }
+
+func short(s string) string {
+	if i := strings.IndexByte(s, ':'); i >= 0 {
+		s = s[i+1:]
+	}
+	if len(s) > 12 {
+		s = s[:12]
+	}
+	return s
+}

internal/agent/agent_test.go

@@ -782,3 +782,30 @@ func TestAStaleRefusedCompositionDoesNotHoldTheNodesOwnBack(t *testing.T) {
 	defer stop()
 	eventually(t, "the node's own not applied", func() bool { return len(b.appliedList()) == 1 })
 }
+
+// A composition the kept declaration or the node's own have says runs is applied all the same when the engine holds no
+// record of its resources: they were removed since (issue 1945, live on 2026-10-25: ace's and g14's node-engines removed
+// ca-trust's, power's and matrix's resources in a node's own apply, and the wants that named those modules' own
+// compositions were judged already running from the kept declaration, so nothing made them again).
+func TestACompositionWhoseResourcesWereRemovedIsAppliedAgain(t *testing.T) {
+	for _, from := range []string{"kept", "have"} {
+		t.Run(from, func(t *testing.T) {
+			b := newBed(t)
+			fp := fpOf("a")
+			rs := []json.RawMessage{file("ca-trust", "trust", "1")}
+			ck, craw, dig := composition(t, "ca-trust", fp, false, rs...)
+			wk, wraw := wantFor(t, "ca-trust", fp, dig, "")
+			b.comps.before(ck, craw)
+			b.wants.before(wk, wraw)
+			if from == "kept" {
+				b.kept = map[string][]json.RawMessage{"ca-trust": rs}
+			} else {
+				b.haves.stored = map[string]Have{"ca-trust": {Module: "ca-trust", FP: fp, Composed: dig, Phase: "running"}}
+			}
+			b.env.Recorded = func(context.Context, delivery.Composed) bool { return false }
+			stop := b.run()
+			defer stop()
+			eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
+		})
+	}
+}

internal/link/have.go

@@ -36,7 +36,10 @@ type haveState struct {
 	mu      sync.Mutex
 	modules map[string]ModuleFact
 	health  map[string]delivery.Health
-	node    *delivery.HaveNode
+	// looked says a look's health was taken: a running module it judges no resource of then has nothing that could be
+	// down, and is ok (issue 1945).
+	looked bool
+	node   *delivery.HaveNode
 	// nodeComposed is the digest of the node's own composition it runs (ADR 0539 §9), kept beside the node's facts.
 	nodeComposed string
 	// written is the build of each module this node-engine wrote a have key for and that the declaration it applied
@@ -110,7 +113,7 @@ func (q *Session) noteApply(r Report) bool {
 		s.mu.Lock()
 		defer s.mu.Unlock()
 		if r.Health != nil {
-			s.health = moduleHealths(r.Health)
+			s.health, s.looked = moduleHealths(r.Health), true
 		}
 		n := nodeFacts(r)
 		s.node = &n
@@ -152,7 +155,7 @@ func (q *Session) noteApply(r Report) bool {
 	}
 	s.modules = next
 	if r.Health != nil {
-		s.health = moduleHealths(r.Health)
+		s.health, s.looked = moduleHealths(r.Health), true
 	}
 	n := nodeFacts(r)
 	s.node = &n
@@ -166,20 +169,20 @@ func (q *Session) noteHealth(h Health) bool {
 	defer s.mu.Unlock()
 	next := moduleHealths(&h)
 	changed := false
-	for module := range s.modules {
+	for module, f := range s.modules {
 		want, ok := next[module]
 		if !ok {
-			want = delivery.HealthUnknown
+			want = nothingJudged(f, true)
 		}
 		have, known := s.health[module]
 		if !known {
-			have = delivery.HealthUnknown
+			have = nothingJudged(f, s.looked)
 		}
 		if have != want {
 			changed = true
 		}
 	}
-	s.health = next
+	s.health, s.looked = next, true
 	return changed
 }
 
@@ -232,6 +235,8 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
 		}
 		if got, known := s.health[module]; known {
 			h.Health = got
+		} else {
+			h.Health = nothingJudged(f, s.looked)
 		}
 		key, err := delivery.HaveKey(node, module)
 		if err != nil {
@@ -452,3 +457,12 @@ func (q *Session) ReadHave(ctx context.Context, module string) (delivery.Have, b
 	}
 	return h, true
 }
+
+// nothingJudged is the health of a module a look judged no resource of: ok when it was looked at and the module applied
+// and runs (nothing of it could be down), unknown otherwise.
+func nothingJudged(f ModuleFact, looked bool) delivery.Health {
+	if looked && !f.Failed && (f.Phase == "" || f.Phase == string(delivery.PhaseRunning)) {
+		return delivery.HealthOK
+	}
+	return delivery.HealthUnknown
+}

internal/link/have_test.go

@@ -498,3 +498,26 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth(
 		t.Fatalf("the node key: %+v %v", n, err)
 	}
 }
+
+// A module applied and running that has nothing long-running to judge is ok once the engine has looked: there is nothing
+// of it that could be down. Unknown kept it from ever being healthy, so the assigner's canary rule held every other node
+// of mesh-host back for good (issue 1945, live on 2026-10-25: ace's have of mesh-host said unknown, and novox and shanks
+// never got the new node-engine). A failed one stays failed, and before any look it is unknown.
+func TestAModuleWithNothingToJudgeIsOkOnceLookedAt(t *testing.T) {
+	l := newHaveLink()
+	q, _ := aHaveQueue(l)
+	q.SayFacts(context.Background(), Report{Modules: []ModuleFact{{Module: "mesh-host", Build: "b"}, {Module: "web", Build: "b"},
+		{Module: "broken", Build: "b", Failed: true, Detail: "x"}}})
+	if h := l.have(t, "n1.mesh-host").Health; h != delivery.HealthUnknown {
+		t.Fatalf("before any look: %s", h)
+	}
+	if !q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateHealthy})) {
+		t.Fatal("not said")
+	}
+	if h := l.have(t, "n1.mesh-host").Health; h != delivery.HealthOK {
+		t.Fatalf("a running module with nothing to judge after a look: %s", h)
+	}
+	if h := l.have(t, "n1.broken").Health; h == delivery.HealthOK {
+		t.Fatalf("a failed module was said ok")
+	}
+}

internal/liveness/liveness.go

@@ -172,6 +172,9 @@ type State struct {
 	Restarts int       `json:"restarts,omitempty"`
 	// Waits is what a resource stated Waiting waits for (ADR 0283).
 	Waits []health.Wait `json:"waits,omitempty"`
+	// Missing is a resource the runtime or the service manager does not have at all after its grace: removed, not
+	// stopped. The node agent applies its module again (issue 1945).
+	Missing bool `json:"missing,omitempty"`
 }
 
 // CheckOf and NeedsOf are a stated resource's declared check, in a word, and the provision it exercises.
@@ -358,7 +361,8 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) {
 		j.judge(k, o, now, r.Kind == KindContainer && held[r.Target], blind, why)
 		seen[r.ID] = true
 		st.Resources = append(st.Resources, State{Resource: k.Resource, State: k.State, Reason: k.Reason, Since: k.Since,
-			Streak: k.Streak, Restarts: k.Counted, Waits: waitsOf(k)})
+			Streak: k.Streak, Restarts: k.Counted, Waits: waitsOf(k),
+			Missing: k.State == Unhealthy && k.Reason == ReasonDown && !blind && !o.Found})
 		// What it waits for is part of what is said: a change of waits alone is a change.
 		word := k.State + "/" + k.Reason + "/" + waitsWord(waitsOf(k))
 		if j.said[r.ID] != word {

internal/liveness/liveness_test.go

@@ -383,3 +383,28 @@ func TestAUserUnitsManagerNotRunningIsNotReadAndNotDown(t *testing.T) {
 		t.Fatalf("the running manager is read in the account's own: %q", last)
 	}
 }
+
+// A resource the runtime does not have at all after its grace is said missing, beside down: what the node agent applies
+// again (issue 1945). One that is there and stopped is down and not missing, and one that cannot be read is neither.
+func TestAResourceNotThereIsMissingAndOneStoppedIsNot(t *testing.T) {
+	c := &clock{now: t0}
+	rt := &fakeRuntime{containers: map[string]Observed{}}
+	j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
+	j.Set([]Resource{server})
+	j.Look(t.Context())
+	c.now = t0.Add(2 * time.Minute)
+	st, _ := j.Look(t.Context())
+	if s := stateOf(t, st, "letta.server"); !s.Missing {
+		t.Fatalf("a container not there after its grace is not missing: %+v", s)
+	}
+	rt.containers["letta-server"] = Observed{Found: true, Identity: "c1"}
+	st, _ = j.Look(t.Context())
+	if s := stateOf(t, st, "letta.server"); s.Missing {
+		t.Fatalf("a container there and stopped: %+v", s)
+	}
+	rt.err = errors.New("cannot connect")
+	st, _ = j.Look(t.Context())
+	if s := stateOf(t, st, "letta.server"); s.Missing {
+		t.Fatalf("a runtime that does not answer says nothing is missing: %+v", s)
+	}
+}