Apply a module again when the engine holds no record of its resources or its health says one is missing, and move a module with nothing to back up without a backup (issue 1944, issue 1945)

2026-10-25 10:59:21 +01:00
parent cf455bc4fe
commit c46937da04
8 changed files with 246 additions and 4 deletions
@@ -0,0 +1,29 @@
+package main
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+// A module that declares nothing to back up on this machine is not backed up before it moves, and so never held for a
+// backup that cannot run (issue 1945, live on 2026-10-25: redis and searxng on ace were held: "redis declares nothing to
+// back up on this machine"). What node-backup.backed-up answers decides it.
+func TestAModuleWithNothingToBackUpIsNotBackedUp(t *testing.T) {
+	for _, c := range []struct {
+		said    string
+		nothing bool
+	}{
+		{`[{"module":"postgres","runs":1,"paths":["/var/lib/postgres"]}]`, true},
+		{`[{"module":"redis","runs":0,"paths":null,"data":[{"item":"cache"}]}]`, true},
+		{`[]`, true},
+		{`[{"module":"redis","runs":1,"paths":["/var/lib/redis"]}]`, false},
+	} {
+		nothing, ok := declaresNothingToBackUp(json.RawMessage(c.said), "redis")
+		if !ok || nothing != c.nothing {
+			t.Errorf("%s: nothing %v (read %v), want %v", c.said, nothing, ok, c.nothing)
+		}
+	}
+	if _, ok := declaresNothingToBackUp(json.RawMessage(`{"error":"x"}`), "redis"); ok {
+		t.Error("an answer that is not the list was read")
+	}
+}
@@ -1166,7 +1166,11 @@ func runLink(ctx context.Context, opts options) error {
 		netJudge.set(network.New(network.Machine{Run: apply.ExecRunner, Linked: queue.Linked},
 			hostOf(mine.Membership.Broker)))
 		go judging.Probe(aside)
-		go judgeWhatRuns(aside, judging, queue, say)
+		go judgeWhatRuns(aside, judging, queue, say, func(modules ...string) {
+			if ag := current.get(); ag != nil {
+				ag.Retry(modules...)
+			}
+		})
 	}
 	// The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the
 	// busy containers are asked for again soon (ADR 0360 §11).
@@ -1329,7 +1333,7 @@ const (
 // the bus when a state changed, again every minute while one is not healthy, and every five minutes
 // anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240
 // rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it.
-func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce) {
+func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce, reapply func(modules ...string)) {
 	ticker := time.NewTicker(liveness.LookEvery)
 	defer ticker.Stop()
 	var lastSaid time.Time
@@ -1415,6 +1419,12 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
 					say(unhealthyWords(r))
 				}
 			}
+			// A resource the machine does not have at all is made again: its module's composition applied anew (issue
+			// 1945), once each time the statement changes, never on a clock.
+			if m := missingModules(st); len(m) > 0 && reapply != nil {
+				say(fmt.Sprintf("applying %s again: a resource it declares is not on this machine", strings.Join(m, ", ")))
+				reapply(m...)
+			}
 		}
 		if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
 			unitSt), accountSt), found), accessSt)) {
@@ -2077,3 +2087,17 @@ func untakenModules(d *declaration.Declaration) map[string][]string {
 	}
 	return d.Adoption.Untaken
 }
+
+// missingModules are the modules of the resources a statement says missing, each once, sorted.
+func missingModules(st liveness.Statement) []string {
+	seen := map[string]bool{}
+	var out []string
+	for _, r := range st.Resources {
+		if r.Missing && r.Module != "" && !seen[r.Module] {
+			seen[r.Module] = true
+			out = append(out, r.Module)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
@@ -105,6 +105,13 @@ func backupNode(session *link.Session) func(ctx context.Context, module string)
 			return agent.Backup{Words: "the end of the backup cannot be heard: " + err.Error()}
 		}
 		defer stop()
+		// A module this machine backs nothing of up moves without a backup: never held for one that cannot run (issue
+		// 1945). An answer that cannot be had or read asks for the backup as before.
+		if said, err := session.AskSeat(ctx, "node-backup", "backed-up", map[string]any{"module": module}, 30*time.Second); err == nil {
+			if nothing, read := declaresNothingToBackUp(said, module); read && nothing {
+				return agent.Backup{Done: true, Words: module + " declares nothing to back up on this machine"}
+			}
+		}
 		asked := time.Now()
 		if _, err := session.AskSeat(ctx, "node-backup", "now", map[string]any{"module": module}, 30*time.Second); err != nil {
 			return agent.Backup{Words: err.Error()}
@@ -286,6 +293,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
 			Composed: bucketWatch{session, composedBucket},
 			Haves:    sessionHaves{session},
 			Kept:     keptByModule(opts, signer),
+			Recorded: recordedIn(opts.state),
 			Trusted:  trustedKeys(opts, signer),
 			Backup:   backupNode(session),
 			Say:      say,
@@ -507,3 +515,44 @@ func refuseEmptyWrites(d *declaration.Declaration) error {
 	}
 	return nil
 }
+
+// recordedIn says the engine's applied state holds a record of every resource of a composition: what the agent believes
+// of a have or the kept declaration only when it does, so a resource removed since is made again (issue 1945). A state
+// that cannot be read says yes: the agent then believes what it was told, as before.
+func recordedIn(statePath string) func(ctx context.Context, c delivery.Composed) bool {
+	return func(_ context.Context, c delivery.Composed) bool {
+		known, err := store.Load(statePath)
+		if err != nil {
+			return true
+		}
+		for _, raw := range c.Resources {
+			r, err := declaration.ParseResource(raw)
+			if err != nil {
+				return true
+			}
+			if _, ok := known.Find(r.Identity()); !ok {
+				return false
+			}
+		}
+		return true
+	}
+}
+
+// declaresNothingToBackUp reads node-backup.backed-up's answer for a module: true when this machine backs nothing of it
+// up (no runs, no paths), and whether the answer could be read at all.
+func declaresNothingToBackUp(said json.RawMessage, module string) (nothing bool, read bool) {
+	var list []struct {
+		Module string   `json:"module"`
+		Runs   int      `json:"runs"`
+		Paths  []string `json:"paths"`
+	}
+	if err := json.Unmarshal(said, &list); err != nil {
+		return false, false
+	}
+	for _, m := range list {
+		if m.Module == module && (m.Runs > 0 || len(m.Paths) > 0) {
+			return false, true
+		}
+	}
+	return true, true
+}
@@ -0,0 +1,48 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"path/filepath"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+
+	"github.com/novox/mesh-host/internal/liveness"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// The record check reads the engine's applied state: a composition one of whose resources has no record there is not
+// recorded (issue 1945).
+func TestACompositionIsRecordedOnlyWhenEveryResourceIs(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "state.json")
+	var s store.State
+	s.Record(store.Applied{ID: "ca-trust.trust", Type: "file", Target: "/etc/ca"})
+	if err := store.Save(path, s); err != nil {
+		t.Fatal(err)
+	}
+	c := delivery.Composed{Node: "ace", Module: "ca-trust", Resources: []json.RawMessage{
+		json.RawMessage(`{"id":"ca-trust.trust","type":"file","module":"ca-trust","path":"/etc/ca","mode":"0644","content":"x"}`)}}
+	recorded := recordedIn(path)
+	if !recorded(context.Background(), c) {
+		t.Fatal("a composition whose every resource has a record was not recorded")
+	}
+	c.Resources = append(c.Resources, json.RawMessage(
+		`{"id":"ca-trust.more","type":"file","module":"ca-trust","path":"/etc/ca2","mode":"0644","content":"y"}`))
+	if recorded(context.Background(), c) {
+		t.Fatal("a composition with a resource the state holds no record of was recorded")
+	}
+}
+
+// The modules applied again are those with a missing resource, each once; one down but there is not among them.
+func TestTheModulesAppliedAgainAreThoseWithAMissingResource(t *testing.T) {
+	st := liveness.Statement{Resources: []liveness.State{
+		{Resource: liveness.Resource{Module: "matrix", ID: "matrix.homeserver"}, State: liveness.Unhealthy, Missing: true},
+		{Resource: liveness.Resource{Module: "matrix", ID: "matrix.db"}, State: liveness.Unhealthy, Missing: true},
+		{Resource: liveness.Resource{Module: "ca-trust", ID: "ca-trust.trust"}, State: liveness.Unhealthy, Missing: true},
+		{Resource: liveness.Resource{Module: "web", ID: "web.server"}, State: liveness.Unhealthy},
+	}}
+	if got := missingModules(st); len(got) != 2 || got[0] != "ca-trust" || got[1] != "matrix" {
+		t.Fatalf("applied again: %v", got)
+	}
+}