Apply a module again when the engine holds no record of its resources or its health says one is missing, and move a module with nothing to back up without a backup (issue 1944, issue 1945)

cmd/mesh-host/backup_nothing_test.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+// A module that declares nothing to back up on this machine is not backed up before it moves, and so never held for a
+// backup that cannot run (issue 1945, live on 2026-10-25: redis and searxng on ace were held: "redis declares nothing to
+// back up on this machine"). What node-backup.backed-up answers decides it.
+func TestAModuleWithNothingToBackUpIsNotBackedUp(t *testing.T) {
+	for _, c := range []struct {
+		said    string
+		nothing bool
+	}{
+		{`[{"module":"postgres","runs":1,"paths":["/var/lib/postgres"]}]`, true},
+		{`[{"module":"redis","runs":0,"paths":null,"data":[{"item":"cache"}]}]`, true},
+		{`[]`, true},
+		{`[{"module":"redis","runs":1,"paths":["/var/lib/redis"]}]`, false},
+	} {
+		nothing, ok := declaresNothingToBackUp(json.RawMessage(c.said), "redis")
+		if !ok || nothing != c.nothing {
+			t.Errorf("%s: nothing %v (read %v), want %v", c.said, nothing, ok, c.nothing)
+		}
+	}
+	if _, ok := declaresNothingToBackUp(json.RawMessage(`{"error":"x"}`), "redis"); ok {
+		t.Error("an answer that is not the list was read")
+	}
+}

cmd/mesh-host/main.go

@@ -1166,7 +1166,11 @@ func runLink(ctx context.Context, opts options) error {
 		netJudge.set(network.New(network.Machine{Run: apply.ExecRunner, Linked: queue.Linked},
 			hostOf(mine.Membership.Broker)))
 		go judging.Probe(aside)
-		go judgeWhatRuns(aside, judging, queue, say)
+		go judgeWhatRuns(aside, judging, queue, say, func(modules ...string) {
+			if ag := current.get(); ag != nil {
+				ag.Retry(modules...)
+			}
+		})
 	}
 	// The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the
 	// busy containers are asked for again soon (ADR 0360 §11).
@@ -1329,7 +1333,7 @@ const (
 // the bus when a state changed, again every minute while one is not healthy, and every five minutes
 // anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240
 // rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it.
-func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce) {
+func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce, reapply func(modules ...string)) {
 	ticker := time.NewTicker(liveness.LookEvery)
 	defer ticker.Stop()
 	var lastSaid time.Time
@@ -1415,6 +1419,12 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
 					say(unhealthyWords(r))
 				}
 			}
+			// A resource the machine does not have at all is made again: its module's composition applied anew (issue
+			// 1945), once each time the statement changes, never on a clock.
+			if m := missingModules(st); len(m) > 0 && reapply != nil {
+				say(fmt.Sprintf("applying %s again: a resource it declares is not on this machine", strings.Join(m, ", ")))
+				reapply(m...)
+			}
 		}
 		if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
 			unitSt), accountSt), found), accessSt)) {
@@ -2077,3 +2087,17 @@ func untakenModules(d *declaration.Declaration) map[string][]string {
 	}
 	return d.Adoption.Untaken
 }
+
+// missingModules are the modules of the resources a statement says missing, each once, sorted.
+func missingModules(st liveness.Statement) []string {
+	seen := map[string]bool{}
+	var out []string
+	for _, r := range st.Resources {
+		if r.Missing && r.Module != "" && !seen[r.Module] {
+			seen[r.Module] = true
+			out = append(out, r.Module)
+		}
+	}
+	sort.Strings(out)
+	return out
+}

cmd/mesh-host/nodeagent.go

@@ -105,6 +105,13 @@ func backupNode(session *link.Session) func(ctx context.Context, module string)
 			return agent.Backup{Words: "the end of the backup cannot be heard: " + err.Error()}
 		}
 		defer stop()
+		// A module this machine backs nothing of up moves without a backup: never held for one that cannot run (issue
+		// 1945). An answer that cannot be had or read asks for the backup as before.
+		if said, err := session.AskSeat(ctx, "node-backup", "backed-up", map[string]any{"module": module}, 30*time.Second); err == nil {
+			if nothing, read := declaresNothingToBackUp(said, module); read && nothing {
+				return agent.Backup{Done: true, Words: module + " declares nothing to back up on this machine"}
+			}
+		}
 		asked := time.Now()
 		if _, err := session.AskSeat(ctx, "node-backup", "now", map[string]any{"module": module}, 30*time.Second); err != nil {
 			return agent.Backup{Words: err.Error()}
@@ -286,6 +293,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
 			Composed: bucketWatch{session, composedBucket},
 			Haves:    sessionHaves{session},
 			Kept:     keptByModule(opts, signer),
+			Recorded: recordedIn(opts.state),
 			Trusted:  trustedKeys(opts, signer),
 			Backup:   backupNode(session),
 			Say:      say,
@@ -507,3 +515,44 @@ func refuseEmptyWrites(d *declaration.Declaration) error {
 	}
 	return nil
 }
+
+// recordedIn says the engine's applied state holds a record of every resource of a composition: what the agent believes
+// of a have or the kept declaration only when it does, so a resource removed since is made again (issue 1945). A state
+// that cannot be read says yes: the agent then believes what it was told, as before.
+func recordedIn(statePath string) func(ctx context.Context, c delivery.Composed) bool {
+	return func(_ context.Context, c delivery.Composed) bool {
+		known, err := store.Load(statePath)
+		if err != nil {
+			return true
+		}
+		for _, raw := range c.Resources {
+			r, err := declaration.ParseResource(raw)
+			if err != nil {
+				return true
+			}
+			if _, ok := known.Find(r.Identity()); !ok {
+				return false
+			}
+		}
+		return true
+	}
+}
+
+// declaresNothingToBackUp reads node-backup.backed-up's answer for a module: true when this machine backs nothing of it
+// up (no runs, no paths), and whether the answer could be read at all.
+func declaresNothingToBackUp(said json.RawMessage, module string) (nothing bool, read bool) {
+	var list []struct {
+		Module string   `json:"module"`
+		Runs   int      `json:"runs"`
+		Paths  []string `json:"paths"`
+	}
+	if err := json.Unmarshal(said, &list); err != nil {
+		return false, false
+	}
+	for _, m := range list {
+		if m.Module == module && (m.Runs > 0 || len(m.Paths) > 0) {
+			return false, true
+		}
+	}
+	return true, true
+}

cmd/mesh-host/recorded_test.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"path/filepath"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+
+	"github.com/novox/mesh-host/internal/liveness"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// The record check reads the engine's applied state: a composition one of whose resources has no record there is not
+// recorded (issue 1945).
+func TestACompositionIsRecordedOnlyWhenEveryResourceIs(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "state.json")
+	var s store.State
+	s.Record(store.Applied{ID: "ca-trust.trust", Type: "file", Target: "/etc/ca"})
+	if err := store.Save(path, s); err != nil {
+		t.Fatal(err)
+	}
+	c := delivery.Composed{Node: "ace", Module: "ca-trust", Resources: []json.RawMessage{
+		json.RawMessage(`{"id":"ca-trust.trust","type":"file","module":"ca-trust","path":"/etc/ca","mode":"0644","content":"x"}`)}}
+	recorded := recordedIn(path)
+	if !recorded(context.Background(), c) {
+		t.Fatal("a composition whose every resource has a record was not recorded")
+	}
+	c.Resources = append(c.Resources, json.RawMessage(
+		`{"id":"ca-trust.more","type":"file","module":"ca-trust","path":"/etc/ca2","mode":"0644","content":"y"}`))
+	if recorded(context.Background(), c) {
+		t.Fatal("a composition with a resource the state holds no record of was recorded")
+	}
+}
+
+// The modules applied again are those with a missing resource, each once; one down but there is not among them.
+func TestTheModulesAppliedAgainAreThoseWithAMissingResource(t *testing.T) {
+	st := liveness.Statement{Resources: []liveness.State{
+		{Resource: liveness.Resource{Module: "matrix", ID: "matrix.homeserver"}, State: liveness.Unhealthy, Missing: true},
+		{Resource: liveness.Resource{Module: "matrix", ID: "matrix.db"}, State: liveness.Unhealthy, Missing: true},
+		{Resource: liveness.Resource{Module: "ca-trust", ID: "ca-trust.trust"}, State: liveness.Unhealthy, Missing: true},
+		{Resource: liveness.Resource{Module: "web", ID: "web.server"}, State: liveness.Unhealthy},
+	}}
+	if got := missingModules(st); len(got) != 2 || got[0] != "ca-trust" || got[1] != "matrix" {
+		t.Fatalf("applied again: %v", got)
+	}
+}

internal/agent/agent.go

@@ -94,7 +94,10 @@ type Env struct {
 	// composition unsigned, signed by another key, or changed after it was signed is refused and nothing of it is
 	// applied. Nil trusts no key, so nothing is applied.
 	Trusted func() []ed25519.PublicKey
-	Say     func(string)
+	// Recorded says the engine holds a record of every resource of the composition, so it was applied and not removed
+	// since; nil says yes. A composition the kept declaration or a have says runs is applied again when it says no.
+	Recorded func(ctx context.Context, c delivery.Composed) bool
+	Say      func(string)
 }
 
 // Agent follows the wants.
@@ -563,6 +566,29 @@ func (a *Agent) settleOne(ctx context.Context, module string) {
 // own have (what this node-engine last applied), else the declaration it kept before the switch (ADR 0539 §9). True when
 // it is, with the have written; nothing is applied.
 func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp compNow) bool {
+	// What a have or the kept declaration says runs is believed only when the engine still holds a record of each of its
+	// resources: a resource removed since (a node's own apply took it, issue 1945) is made again by an apply.
+	if a.env.Recorded != nil && !a.env.Recorded(ctx, comp.C) {
+		if module != NodeModule {
+			if _, ok := a.env.Haves.Read(ctx, module); ok {
+				a.mu.Lock()
+				a.prior[module] = true
+				a.mu.Unlock()
+			}
+		}
+		if a.env.Kept != nil {
+			if _, ok := a.env.Kept()[module]; ok {
+				a.mu.Lock()
+				a.prior[module] = true
+				a.mu.Unlock()
+			}
+		}
+		a.mu.Lock()
+		a.sayOnce("unrecorded:"+module+":"+comp.Dig, fmt.Sprintf("%s's composition %s is applied again: this engine holds "+
+			"no record of some of its resources", module, short(comp.Dig)))
+		a.mu.Unlock()
+		return false
+	}
 	if module != NodeModule {
 		if h, ok := a.env.Haves.Read(ctx, module); ok {
 			a.mu.Lock()
@@ -718,3 +744,13 @@ func Canonical(raw json.RawMessage) (json.RawMessage, error) {
 	}
 	return json.Marshal(v)
 }
+
+func short(s string) string {
+	if i := strings.IndexByte(s, ':'); i >= 0 {
+		s = s[i+1:]
+	}
+	if len(s) > 12 {
+		s = s[:12]
+	}
+	return s
+}

internal/agent/agent_test.go

@@ -782,3 +782,30 @@ func TestAStaleRefusedCompositionDoesNotHoldTheNodesOwnBack(t *testing.T) {
 	defer stop()
 	eventually(t, "the node's own not applied", func() bool { return len(b.appliedList()) == 1 })
 }
+
+// A composition the kept declaration or the node's own have says runs is applied all the same when the engine holds no
+// record of its resources: they were removed since (issue 1945, live on 2026-10-25: ace's and g14's node-engines removed
+// ca-trust's, power's and matrix's resources in a node's own apply, and the wants that named those modules' own
+// compositions were judged already running from the kept declaration, so nothing made them again).
+func TestACompositionWhoseResourcesWereRemovedIsAppliedAgain(t *testing.T) {
+	for _, from := range []string{"kept", "have"} {
+		t.Run(from, func(t *testing.T) {
+			b := newBed(t)
+			fp := fpOf("a")
+			rs := []json.RawMessage{file("ca-trust", "trust", "1")}
+			ck, craw, dig := composition(t, "ca-trust", fp, false, rs...)
+			wk, wraw := wantFor(t, "ca-trust", fp, dig, "")
+			b.comps.before(ck, craw)
+			b.wants.before(wk, wraw)
+			if from == "kept" {
+				b.kept = map[string][]json.RawMessage{"ca-trust": rs}
+			} else {
+				b.haves.stored = map[string]Have{"ca-trust": {Module: "ca-trust", FP: fp, Composed: dig, Phase: "running"}}
+			}
+			b.env.Recorded = func(context.Context, delivery.Composed) bool { return false }
+			stop := b.run()
+			defer stop()
+			eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
+		})
+	}
+}

internal/liveness/liveness.go

@@ -172,6 +172,9 @@ type State struct {
 	Restarts int       `json:"restarts,omitempty"`
 	// Waits is what a resource stated Waiting waits for (ADR 0283).
 	Waits []health.Wait `json:"waits,omitempty"`
+	// Missing is a resource the runtime or the service manager does not have at all after its grace: removed, not
+	// stopped. The node agent applies its module again (issue 1945).
+	Missing bool `json:"missing,omitempty"`
 }
 
 // CheckOf and NeedsOf are a stated resource's declared check, in a word, and the provision it exercises.
@@ -358,7 +361,8 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) {
 		j.judge(k, o, now, r.Kind == KindContainer && held[r.Target], blind, why)
 		seen[r.ID] = true
 		st.Resources = append(st.Resources, State{Resource: k.Resource, State: k.State, Reason: k.Reason, Since: k.Since,
-			Streak: k.Streak, Restarts: k.Counted, Waits: waitsOf(k)})
+			Streak: k.Streak, Restarts: k.Counted, Waits: waitsOf(k),
+			Missing: k.State == Unhealthy && k.Reason == ReasonDown && !blind && !o.Found})
 		// What it waits for is part of what is said: a change of waits alone is a change.
 		word := k.State + "/" + k.Reason + "/" + waitsWord(waitsOf(k))
 		if j.said[r.ID] != word {

internal/liveness/liveness_test.go

@@ -383,3 +383,28 @@ func TestAUserUnitsManagerNotRunningIsNotReadAndNotDown(t *testing.T) {
 		t.Fatalf("the running manager is read in the account's own: %q", last)
 	}
 }
+
+// A resource the runtime does not have at all after its grace is said missing, beside down: what the node agent applies
+// again (issue 1945). One that is there and stopped is down and not missing, and one that cannot be read is neither.
+func TestAResourceNotThereIsMissingAndOneStoppedIsNot(t *testing.T) {
+	c := &clock{now: t0}
+	rt := &fakeRuntime{containers: map[string]Observed{}}
+	j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
+	j.Set([]Resource{server})
+	j.Look(t.Context())
+	c.now = t0.Add(2 * time.Minute)
+	st, _ := j.Look(t.Context())
+	if s := stateOf(t, st, "letta.server"); !s.Missing {
+		t.Fatalf("a container not there after its grace is not missing: %+v", s)
+	}
+	rt.containers["letta-server"] = Observed{Found: true, Identity: "c1"}
+	st, _ = j.Look(t.Context())
+	if s := stateOf(t, st, "letta.server"); s.Missing {
+		t.Fatalf("a container there and stopped: %+v", s)
+	}
+	rt.err = errors.New("cannot connect")
+	st, _ = j.Look(t.Context())
+	if s := stateOf(t, st, "letta.server"); s.Missing {
+		t.Fatalf("a runtime that does not answer says nothing is missing: %+v", s)
+	}
+}