From c46937da04e324dabc590eff07fc0881b0d05ca1 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 25 Oct 2026 10:59:21 +0100 Subject: [PATCH] Apply a module again when the engine holds no record of its resources or its health says one is missing, and move a module with nothing to back up without a backup (issue 1944, issue 1945) --- cmd/mesh-host/backup_nothing_test.go | 29 ++++++++++++++++ cmd/mesh-host/main.go | 28 ++++++++++++++-- cmd/mesh-host/nodeagent.go | 49 ++++++++++++++++++++++++++++ cmd/mesh-host/recorded_test.go | 48 +++++++++++++++++++++++++++ internal/agent/agent.go | 38 ++++++++++++++++++++- internal/agent/agent_test.go | 27 +++++++++++++++ internal/liveness/liveness.go | 6 +++- internal/liveness/liveness_test.go | 25 ++++++++++++++ 8 files changed, 246 insertions(+), 4 deletions(-) create mode 100644 cmd/mesh-host/backup_nothing_test.go create mode 100644 cmd/mesh-host/recorded_test.go diff --git a/cmd/mesh-host/backup_nothing_test.go b/cmd/mesh-host/backup_nothing_test.go new file mode 100644 index 0000000..bae021e --- /dev/null +++ b/cmd/mesh-host/backup_nothing_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "encoding/json" + "testing" +) + +// A module that declares nothing to back up on this machine is not backed up before it moves, and so never held for a +// backup that cannot run (issue 1945, live on 2026-10-25: redis and searxng on ace were held: "redis declares nothing to +// back up on this machine"). What node-backup.backed-up answers decides it. +func TestAModuleWithNothingToBackUpIsNotBackedUp(t *testing.T) { + for _, c := range []struct { + said string + nothing bool + }{ + {`[{"module":"postgres","runs":1,"paths":["/var/lib/postgres"]}]`, true}, + {`[{"module":"redis","runs":0,"paths":null,"data":[{"item":"cache"}]}]`, true}, + {`[]`, true}, + {`[{"module":"redis","runs":1,"paths":["/var/lib/redis"]}]`, false}, + } { + nothing, ok := declaresNothingToBackUp(json.RawMessage(c.said), "redis") + if !ok || nothing != c.nothing { + t.Errorf("%s: nothing %v (read %v), want %v", c.said, nothing, ok, c.nothing) + } + } + if _, ok := declaresNothingToBackUp(json.RawMessage(`{"error":"x"}`), "redis"); ok { + t.Error("an answer that is not the list was read") + } +} diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index eb54e8b..01ae6ba 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1166,7 +1166,11 @@ func runLink(ctx context.Context, opts options) error { netJudge.set(network.New(network.Machine{Run: apply.ExecRunner, Linked: queue.Linked}, hostOf(mine.Membership.Broker))) go judging.Probe(aside) - go judgeWhatRuns(aside, judging, queue, say) + go judgeWhatRuns(aside, judging, queue, say, func(modules ...string) { + if ag := current.get(); ag != nil { + ag.Retry(modules...) + } + }) } // The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the // busy containers are asked for again soon (ADR 0360 §11). @@ -1329,7 +1333,7 @@ const ( // the bus when a state changed, again every minute while one is not healthy, and every five minutes // anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240 // rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it. -func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce) { +func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce, reapply func(modules ...string)) { ticker := time.NewTicker(liveness.LookEvery) defer ticker.Stop() var lastSaid time.Time @@ -1415,6 +1419,12 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say(unhealthyWords(r)) } } + // A resource the machine does not have at all is made again: its module's composition applied anew (issue + // 1945), once each time the statement changes, never on a clock. + if m := missingModules(st); len(m) > 0 && reapply != nil { + say(fmt.Sprintf("applying %s again: a resource it declares is not on this machine", strings.Join(m, ", "))) + reapply(m...) + } } if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt), accountSt), found), accessSt)) { @@ -2077,3 +2087,17 @@ func untakenModules(d *declaration.Declaration) map[string][]string { } return d.Adoption.Untaken } + +// missingModules are the modules of the resources a statement says missing, each once, sorted. +func missingModules(st liveness.Statement) []string { + seen := map[string]bool{} + var out []string + for _, r := range st.Resources { + if r.Missing && r.Module != "" && !seen[r.Module] { + seen[r.Module] = true + out = append(out, r.Module) + } + } + sort.Strings(out) + return out +} diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 906ddba..e4e902e 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -105,6 +105,13 @@ func backupNode(session *link.Session) func(ctx context.Context, module string) return agent.Backup{Words: "the end of the backup cannot be heard: " + err.Error()} } defer stop() + // A module this machine backs nothing of up moves without a backup: never held for one that cannot run (issue + // 1945). An answer that cannot be had or read asks for the backup as before. + if said, err := session.AskSeat(ctx, "node-backup", "backed-up", map[string]any{"module": module}, 30*time.Second); err == nil { + if nothing, read := declaresNothingToBackUp(said, module); read && nothing { + return agent.Backup{Done: true, Words: module + " declares nothing to back up on this machine"} + } + } asked := time.Now() if _, err := session.AskSeat(ctx, "node-backup", "now", map[string]any{"module": module}, 30*time.Second); err != nil { return agent.Backup{Words: err.Error()} @@ -286,6 +293,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess Composed: bucketWatch{session, composedBucket}, Haves: sessionHaves{session}, Kept: keptByModule(opts, signer), + Recorded: recordedIn(opts.state), Trusted: trustedKeys(opts, signer), Backup: backupNode(session), Say: say, @@ -507,3 +515,44 @@ func refuseEmptyWrites(d *declaration.Declaration) error { } return nil } + +// recordedIn says the engine's applied state holds a record of every resource of a composition: what the agent believes +// of a have or the kept declaration only when it does, so a resource removed since is made again (issue 1945). A state +// that cannot be read says yes: the agent then believes what it was told, as before. +func recordedIn(statePath string) func(ctx context.Context, c delivery.Composed) bool { + return func(_ context.Context, c delivery.Composed) bool { + known, err := store.Load(statePath) + if err != nil { + return true + } + for _, raw := range c.Resources { + r, err := declaration.ParseResource(raw) + if err != nil { + return true + } + if _, ok := known.Find(r.Identity()); !ok { + return false + } + } + return true + } +} + +// declaresNothingToBackUp reads node-backup.backed-up's answer for a module: true when this machine backs nothing of it +// up (no runs, no paths), and whether the answer could be read at all. +func declaresNothingToBackUp(said json.RawMessage, module string) (nothing bool, read bool) { + var list []struct { + Module string `json:"module"` + Runs int `json:"runs"` + Paths []string `json:"paths"` + } + if err := json.Unmarshal(said, &list); err != nil { + return false, false + } + for _, m := range list { + if m.Module == module && (m.Runs > 0 || len(m.Paths) > 0) { + return false, true + } + } + return true, true +} diff --git a/cmd/mesh-host/recorded_test.go b/cmd/mesh-host/recorded_test.go new file mode 100644 index 0000000..bba28e2 --- /dev/null +++ b/cmd/mesh-host/recorded_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "context" + "encoding/json" + "path/filepath" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" + + "github.com/novox/mesh-host/internal/liveness" + "github.com/novox/mesh-host/internal/store" +) + +// The record check reads the engine's applied state: a composition one of whose resources has no record there is not +// recorded (issue 1945). +func TestACompositionIsRecordedOnlyWhenEveryResourceIs(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + var s store.State + s.Record(store.Applied{ID: "ca-trust.trust", Type: "file", Target: "/etc/ca"}) + if err := store.Save(path, s); err != nil { + t.Fatal(err) + } + c := delivery.Composed{Node: "ace", Module: "ca-trust", Resources: []json.RawMessage{ + json.RawMessage(`{"id":"ca-trust.trust","type":"file","module":"ca-trust","path":"/etc/ca","mode":"0644","content":"x"}`)}} + recorded := recordedIn(path) + if !recorded(context.Background(), c) { + t.Fatal("a composition whose every resource has a record was not recorded") + } + c.Resources = append(c.Resources, json.RawMessage( + `{"id":"ca-trust.more","type":"file","module":"ca-trust","path":"/etc/ca2","mode":"0644","content":"y"}`)) + if recorded(context.Background(), c) { + t.Fatal("a composition with a resource the state holds no record of was recorded") + } +} + +// The modules applied again are those with a missing resource, each once; one down but there is not among them. +func TestTheModulesAppliedAgainAreThoseWithAMissingResource(t *testing.T) { + st := liveness.Statement{Resources: []liveness.State{ + {Resource: liveness.Resource{Module: "matrix", ID: "matrix.homeserver"}, State: liveness.Unhealthy, Missing: true}, + {Resource: liveness.Resource{Module: "matrix", ID: "matrix.db"}, State: liveness.Unhealthy, Missing: true}, + {Resource: liveness.Resource{Module: "ca-trust", ID: "ca-trust.trust"}, State: liveness.Unhealthy, Missing: true}, + {Resource: liveness.Resource{Module: "web", ID: "web.server"}, State: liveness.Unhealthy}, + }} + if got := missingModules(st); len(got) != 2 || got[0] != "ca-trust" || got[1] != "matrix" { + t.Fatalf("applied again: %v", got) + } +} diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 40f6fe4..b5350e8 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -94,7 +94,10 @@ type Env struct { // composition unsigned, signed by another key, or changed after it was signed is refused and nothing of it is // applied. Nil trusts no key, so nothing is applied. Trusted func() []ed25519.PublicKey - Say func(string) + // Recorded says the engine holds a record of every resource of the composition, so it was applied and not removed + // since; nil says yes. A composition the kept declaration or a have says runs is applied again when it says no. + Recorded func(ctx context.Context, c delivery.Composed) bool + Say func(string) } // Agent follows the wants. @@ -563,6 +566,29 @@ func (a *Agent) settleOne(ctx context.Context, module string) { // own have (what this node-engine last applied), else the declaration it kept before the switch (ADR 0539 §9). True when // it is, with the have written; nothing is applied. func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp compNow) bool { + // What a have or the kept declaration says runs is believed only when the engine still holds a record of each of its + // resources: a resource removed since (a node's own apply took it, issue 1945) is made again by an apply. + if a.env.Recorded != nil && !a.env.Recorded(ctx, comp.C) { + if module != NodeModule { + if _, ok := a.env.Haves.Read(ctx, module); ok { + a.mu.Lock() + a.prior[module] = true + a.mu.Unlock() + } + } + if a.env.Kept != nil { + if _, ok := a.env.Kept()[module]; ok { + a.mu.Lock() + a.prior[module] = true + a.mu.Unlock() + } + } + a.mu.Lock() + a.sayOnce("unrecorded:"+module+":"+comp.Dig, fmt.Sprintf("%s's composition %s is applied again: this engine holds "+ + "no record of some of its resources", module, short(comp.Dig))) + a.mu.Unlock() + return false + } if module != NodeModule { if h, ok := a.env.Haves.Read(ctx, module); ok { a.mu.Lock() @@ -718,3 +744,13 @@ func Canonical(raw json.RawMessage) (json.RawMessage, error) { } return json.Marshal(v) } + +func short(s string) string { + if i := strings.IndexByte(s, ':'); i >= 0 { + s = s[i+1:] + } + if len(s) > 12 { + s = s[:12] + } + return s +} diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 793d9a3..320cb19 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -782,3 +782,30 @@ func TestAStaleRefusedCompositionDoesNotHoldTheNodesOwnBack(t *testing.T) { defer stop() eventually(t, "the node's own not applied", func() bool { return len(b.appliedList()) == 1 }) } + +// A composition the kept declaration or the node's own have says runs is applied all the same when the engine holds no +// record of its resources: they were removed since (issue 1945, live on 2026-10-25: ace's and g14's node-engines removed +// ca-trust's, power's and matrix's resources in a node's own apply, and the wants that named those modules' own +// compositions were judged already running from the kept declaration, so nothing made them again). +func TestACompositionWhoseResourcesWereRemovedIsAppliedAgain(t *testing.T) { + for _, from := range []string{"kept", "have"} { + t.Run(from, func(t *testing.T) { + b := newBed(t) + fp := fpOf("a") + rs := []json.RawMessage{file("ca-trust", "trust", "1")} + ck, craw, dig := composition(t, "ca-trust", fp, false, rs...) + wk, wraw := wantFor(t, "ca-trust", fp, dig, "") + b.comps.before(ck, craw) + b.wants.before(wk, wraw) + if from == "kept" { + b.kept = map[string][]json.RawMessage{"ca-trust": rs} + } else { + b.haves.stored = map[string]Have{"ca-trust": {Module: "ca-trust", FP: fp, Composed: dig, Phase: "running"}} + } + b.env.Recorded = func(context.Context, delivery.Composed) bool { return false } + stop := b.run() + defer stop() + eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 }) + }) + } +} diff --git a/internal/liveness/liveness.go b/internal/liveness/liveness.go index 2ae6c9e..9491471 100644 --- a/internal/liveness/liveness.go +++ b/internal/liveness/liveness.go @@ -172,6 +172,9 @@ type State struct { Restarts int `json:"restarts,omitempty"` // Waits is what a resource stated Waiting waits for (ADR 0283). Waits []health.Wait `json:"waits,omitempty"` + // Missing is a resource the runtime or the service manager does not have at all after its grace: removed, not + // stopped. The node agent applies its module again (issue 1945). + Missing bool `json:"missing,omitempty"` } // CheckOf and NeedsOf are a stated resource's declared check, in a word, and the provision it exercises. @@ -358,7 +361,8 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) { j.judge(k, o, now, r.Kind == KindContainer && held[r.Target], blind, why) seen[r.ID] = true st.Resources = append(st.Resources, State{Resource: k.Resource, State: k.State, Reason: k.Reason, Since: k.Since, - Streak: k.Streak, Restarts: k.Counted, Waits: waitsOf(k)}) + Streak: k.Streak, Restarts: k.Counted, Waits: waitsOf(k), + Missing: k.State == Unhealthy && k.Reason == ReasonDown && !blind && !o.Found}) // What it waits for is part of what is said: a change of waits alone is a change. word := k.State + "/" + k.Reason + "/" + waitsWord(waitsOf(k)) if j.said[r.ID] != word { diff --git a/internal/liveness/liveness_test.go b/internal/liveness/liveness_test.go index 589dd16..74b4764 100644 --- a/internal/liveness/liveness_test.go +++ b/internal/liveness/liveness_test.go @@ -383,3 +383,28 @@ func TestAUserUnitsManagerNotRunningIsNotReadAndNotDown(t *testing.T) { t.Fatalf("the running manager is read in the account's own: %q", last) } } + +// A resource the runtime does not have at all after its grace is said missing, beside down: what the node agent applies +// again (issue 1945). One that is there and stopped is down and not missing, and one that cannot be read is neither. +func TestAResourceNotThereIsMissingAndOneStoppedIsNot(t *testing.T) { + c := &clock{now: t0} + rt := &fakeRuntime{containers: map[string]Observed{}} + j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName)) + j.Set([]Resource{server}) + j.Look(t.Context()) + c.now = t0.Add(2 * time.Minute) + st, _ := j.Look(t.Context()) + if s := stateOf(t, st, "letta.server"); !s.Missing { + t.Fatalf("a container not there after its grace is not missing: %+v", s) + } + rt.containers["letta-server"] = Observed{Found: true, Identity: "c1"} + st, _ = j.Look(t.Context()) + if s := stateOf(t, st, "letta.server"); s.Missing { + t.Fatalf("a container there and stopped: %+v", s) + } + rt.err = errors.New("cannot connect") + st, _ = j.Look(t.Context()) + if s := stateOf(t, st, "letta.server"); s.Missing { + t.Fatalf("a runtime that does not answer says nothing is missing: %+v", s) + } +}