Apply a module again when the engine holds no record of its resources or its health says one is missing, and move a module with nothing to back up without a backup (issue 1944, issue 1945)

internal/agent/agent.go

@@ -94,7 +94,10 @@ type Env struct {
 	// composition unsigned, signed by another key, or changed after it was signed is refused and nothing of it is
 	// applied. Nil trusts no key, so nothing is applied.
 	Trusted func() []ed25519.PublicKey
-	Say     func(string)
+	// Recorded says the engine holds a record of every resource of the composition, so it was applied and not removed
+	// since; nil says yes. A composition the kept declaration or a have says runs is applied again when it says no.
+	Recorded func(ctx context.Context, c delivery.Composed) bool
+	Say      func(string)
 }
 
 // Agent follows the wants.
@@ -563,6 +566,29 @@ func (a *Agent) settleOne(ctx context.Context, module string) {
 // own have (what this node-engine last applied), else the declaration it kept before the switch (ADR 0539 §9). True when
 // it is, with the have written; nothing is applied.
 func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp compNow) bool {
+	// What a have or the kept declaration says runs is believed only when the engine still holds a record of each of its
+	// resources: a resource removed since (a node's own apply took it, issue 1945) is made again by an apply.
+	if a.env.Recorded != nil && !a.env.Recorded(ctx, comp.C) {
+		if module != NodeModule {
+			if _, ok := a.env.Haves.Read(ctx, module); ok {
+				a.mu.Lock()
+				a.prior[module] = true
+				a.mu.Unlock()
+			}
+		}
+		if a.env.Kept != nil {
+			if _, ok := a.env.Kept()[module]; ok {
+				a.mu.Lock()
+				a.prior[module] = true
+				a.mu.Unlock()
+			}
+		}
+		a.mu.Lock()
+		a.sayOnce("unrecorded:"+module+":"+comp.Dig, fmt.Sprintf("%s's composition %s is applied again: this engine holds "+
+			"no record of some of its resources", module, short(comp.Dig)))
+		a.mu.Unlock()
+		return false
+	}
 	if module != NodeModule {
 		if h, ok := a.env.Haves.Read(ctx, module); ok {
 			a.mu.Lock()
@@ -718,3 +744,13 @@ func Canonical(raw json.RawMessage) (json.RawMessage, error) {
 	}
 	return json.Marshal(v)
 }
+
+func short(s string) string {
+	if i := strings.IndexByte(s, ':'); i >= 0 {
+		s = s[i+1:]
+	}
+	if len(s) > 12 {
+		s = s[:12]
+	}
+	return s
+}

internal/agent/agent_test.go

@@ -782,3 +782,30 @@ func TestAStaleRefusedCompositionDoesNotHoldTheNodesOwnBack(t *testing.T) {
 	defer stop()
 	eventually(t, "the node's own not applied", func() bool { return len(b.appliedList()) == 1 })
 }
+
+// A composition the kept declaration or the node's own have says runs is applied all the same when the engine holds no
+// record of its resources: they were removed since (issue 1945, live on 2026-10-25: ace's and g14's node-engines removed
+// ca-trust's, power's and matrix's resources in a node's own apply, and the wants that named those modules' own
+// compositions were judged already running from the kept declaration, so nothing made them again).
+func TestACompositionWhoseResourcesWereRemovedIsAppliedAgain(t *testing.T) {
+	for _, from := range []string{"kept", "have"} {
+		t.Run(from, func(t *testing.T) {
+			b := newBed(t)
+			fp := fpOf("a")
+			rs := []json.RawMessage{file("ca-trust", "trust", "1")}
+			ck, craw, dig := composition(t, "ca-trust", fp, false, rs...)
+			wk, wraw := wantFor(t, "ca-trust", fp, dig, "")
+			b.comps.before(ck, craw)
+			b.wants.before(wk, wraw)
+			if from == "kept" {
+				b.kept = map[string][]json.RawMessage{"ca-trust": rs}
+			} else {
+				b.haves.stored = map[string]Have{"ca-trust": {Module: "ca-trust", FP: fp, Composed: dig, Phase: "running"}}
+			}
+			b.env.Recorded = func(context.Context, delivery.Composed) bool { return false }
+			stop := b.run()
+			defer stop()
+			eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
+		})
+	}
+}

internal/liveness/liveness.go

@@ -172,6 +172,9 @@ type State struct {
 	Restarts int       `json:"restarts,omitempty"`
 	// Waits is what a resource stated Waiting waits for (ADR 0283).
 	Waits []health.Wait `json:"waits,omitempty"`
+	// Missing is a resource the runtime or the service manager does not have at all after its grace: removed, not
+	// stopped. The node agent applies its module again (issue 1945).
+	Missing bool `json:"missing,omitempty"`
 }
 
 // CheckOf and NeedsOf are a stated resource's declared check, in a word, and the provision it exercises.
@@ -358,7 +361,8 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) {
 		j.judge(k, o, now, r.Kind == KindContainer && held[r.Target], blind, why)
 		seen[r.ID] = true
 		st.Resources = append(st.Resources, State{Resource: k.Resource, State: k.State, Reason: k.Reason, Since: k.Since,
-			Streak: k.Streak, Restarts: k.Counted, Waits: waitsOf(k)})
+			Streak: k.Streak, Restarts: k.Counted, Waits: waitsOf(k),
+			Missing: k.State == Unhealthy && k.Reason == ReasonDown && !blind && !o.Found})
 		// What it waits for is part of what is said: a change of waits alone is a change.
 		word := k.State + "/" + k.Reason + "/" + waitsWord(waitsOf(k))
 		if j.said[r.ID] != word {

internal/liveness/liveness_test.go

@@ -383,3 +383,28 @@ func TestAUserUnitsManagerNotRunningIsNotReadAndNotDown(t *testing.T) {
 		t.Fatalf("the running manager is read in the account's own: %q", last)
 	}
 }
+
+// A resource the runtime does not have at all after its grace is said missing, beside down: what the node agent applies
+// again (issue 1945). One that is there and stopped is down and not missing, and one that cannot be read is neither.
+func TestAResourceNotThereIsMissingAndOneStoppedIsNot(t *testing.T) {
+	c := &clock{now: t0}
+	rt := &fakeRuntime{containers: map[string]Observed{}}
+	j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
+	j.Set([]Resource{server})
+	j.Look(t.Context())
+	c.now = t0.Add(2 * time.Minute)
+	st, _ := j.Look(t.Context())
+	if s := stateOf(t, st, "letta.server"); !s.Missing {
+		t.Fatalf("a container not there after its grace is not missing: %+v", s)
+	}
+	rt.containers["letta-server"] = Observed{Found: true, Identity: "c1"}
+	st, _ = j.Look(t.Context())
+	if s := stateOf(t, st, "letta.server"); s.Missing {
+		t.Fatalf("a container there and stopped: %+v", s)
+	}
+	rt.err = errors.New("cannot connect")
+	st, _ = j.Look(t.Context())
+	if s := stateOf(t, st, "letta.server"); s.Missing {
+		t.Fatalf("a runtime that does not answer says nothing is missing: %+v", s)
+	}
+}