From c46937da04e324dabc590eff07fc0881b0d05ca1 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 25 Oct 2026 10:59:21 +0100 Subject: [PATCH 1/3] Apply a module again when the engine holds no record of its resources or its health says one is missing, and move a module with nothing to back up without a backup (issue 1944, issue 1945) --- cmd/mesh-host/backup_nothing_test.go | 29 ++++++++++++++++ cmd/mesh-host/main.go | 28 ++++++++++++++-- cmd/mesh-host/nodeagent.go | 49 ++++++++++++++++++++++++++++ cmd/mesh-host/recorded_test.go | 48 +++++++++++++++++++++++++++ internal/agent/agent.go | 38 ++++++++++++++++++++- internal/agent/agent_test.go | 27 +++++++++++++++ internal/liveness/liveness.go | 6 +++- internal/liveness/liveness_test.go | 25 ++++++++++++++ 8 files changed, 246 insertions(+), 4 deletions(-) create mode 100644 cmd/mesh-host/backup_nothing_test.go create mode 100644 cmd/mesh-host/recorded_test.go diff --git a/cmd/mesh-host/backup_nothing_test.go b/cmd/mesh-host/backup_nothing_test.go new file mode 100644 index 0000000..bae021e --- /dev/null +++ b/cmd/mesh-host/backup_nothing_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "encoding/json" + "testing" +) + +// A module that declares nothing to back up on this machine is not backed up before it moves, and so never held for a +// backup that cannot run (issue 1945, live on 2026-10-25: redis and searxng on ace were held: "redis declares nothing to +// back up on this machine"). What node-backup.backed-up answers decides it. +func TestAModuleWithNothingToBackUpIsNotBackedUp(t *testing.T) { + for _, c := range []struct { + said string + nothing bool + }{ + {`[{"module":"postgres","runs":1,"paths":["/var/lib/postgres"]}]`, true}, + {`[{"module":"redis","runs":0,"paths":null,"data":[{"item":"cache"}]}]`, true}, + {`[]`, true}, + {`[{"module":"redis","runs":1,"paths":["/var/lib/redis"]}]`, false}, + } { + nothing, ok := declaresNothingToBackUp(json.RawMessage(c.said), "redis") + if !ok || nothing != c.nothing { + t.Errorf("%s: nothing %v (read %v), want %v", c.said, nothing, ok, c.nothing) + } + } + if _, ok := declaresNothingToBackUp(json.RawMessage(`{"error":"x"}`), "redis"); ok { + t.Error("an answer that is not the list was read") + } +} diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index eb54e8b..01ae6ba 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1166,7 +1166,11 @@ func runLink(ctx context.Context, opts options) error { netJudge.set(network.New(network.Machine{Run: apply.ExecRunner, Linked: queue.Linked}, hostOf(mine.Membership.Broker))) go judging.Probe(aside) - go judgeWhatRuns(aside, judging, queue, say) + go judgeWhatRuns(aside, judging, queue, say, func(modules ...string) { + if ag := current.get(); ag != nil { + ag.Retry(modules...) + } + }) } // The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the // busy containers are asked for again soon (ADR 0360 §11). @@ -1329,7 +1333,7 @@ const ( // the bus when a state changed, again every minute while one is not healthy, and every five minutes // anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240 // rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it. -func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce) { +func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce, reapply func(modules ...string)) { ticker := time.NewTicker(liveness.LookEvery) defer ticker.Stop() var lastSaid time.Time @@ -1415,6 +1419,12 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say(unhealthyWords(r)) } } + // A resource the machine does not have at all is made again: its module's composition applied anew (issue + // 1945), once each time the statement changes, never on a clock. + if m := missingModules(st); len(m) > 0 && reapply != nil { + say(fmt.Sprintf("applying %s again: a resource it declares is not on this machine", strings.Join(m, ", "))) + reapply(m...) + } } if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt), accountSt), found), accessSt)) { @@ -2077,3 +2087,17 @@ func untakenModules(d *declaration.Declaration) map[string][]string { } return d.Adoption.Untaken } + +// missingModules are the modules of the resources a statement says missing, each once, sorted. +func missingModules(st liveness.Statement) []string { + seen := map[string]bool{} + var out []string + for _, r := range st.Resources { + if r.Missing && r.Module != "" && !seen[r.Module] { + seen[r.Module] = true + out = append(out, r.Module) + } + } + sort.Strings(out) + return out +} diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 906ddba..e4e902e 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -105,6 +105,13 @@ func backupNode(session *link.Session) func(ctx context.Context, module string) return agent.Backup{Words: "the end of the backup cannot be heard: " + err.Error()} } defer stop() + // A module this machine backs nothing of up moves without a backup: never held for one that cannot run (issue + // 1945). An answer that cannot be had or read asks for the backup as before. + if said, err := session.AskSeat(ctx, "node-backup", "backed-up", map[string]any{"module": module}, 30*time.Second); err == nil { + if nothing, read := declaresNothingToBackUp(said, module); read && nothing { + return agent.Backup{Done: true, Words: module + " declares nothing to back up on this machine"} + } + } asked := time.Now() if _, err := session.AskSeat(ctx, "node-backup", "now", map[string]any{"module": module}, 30*time.Second); err != nil { return agent.Backup{Words: err.Error()} @@ -286,6 +293,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess Composed: bucketWatch{session, composedBucket}, Haves: sessionHaves{session}, Kept: keptByModule(opts, signer), + Recorded: recordedIn(opts.state), Trusted: trustedKeys(opts, signer), Backup: backupNode(session), Say: say, @@ -507,3 +515,44 @@ func refuseEmptyWrites(d *declaration.Declaration) error { } return nil } + +// recordedIn says the engine's applied state holds a record of every resource of a composition: what the agent believes +// of a have or the kept declaration only when it does, so a resource removed since is made again (issue 1945). A state +// that cannot be read says yes: the agent then believes what it was told, as before. +func recordedIn(statePath string) func(ctx context.Context, c delivery.Composed) bool { + return func(_ context.Context, c delivery.Composed) bool { + known, err := store.Load(statePath) + if err != nil { + return true + } + for _, raw := range c.Resources { + r, err := declaration.ParseResource(raw) + if err != nil { + return true + } + if _, ok := known.Find(r.Identity()); !ok { + return false + } + } + return true + } +} + +// declaresNothingToBackUp reads node-backup.backed-up's answer for a module: true when this machine backs nothing of it +// up (no runs, no paths), and whether the answer could be read at all. +func declaresNothingToBackUp(said json.RawMessage, module string) (nothing bool, read bool) { + var list []struct { + Module string `json:"module"` + Runs int `json:"runs"` + Paths []string `json:"paths"` + } + if err := json.Unmarshal(said, &list); err != nil { + return false, false + } + for _, m := range list { + if m.Module == module && (m.Runs > 0 || len(m.Paths) > 0) { + return false, true + } + } + return true, true +} diff --git a/cmd/mesh-host/recorded_test.go b/cmd/mesh-host/recorded_test.go new file mode 100644 index 0000000..bba28e2 --- /dev/null +++ b/cmd/mesh-host/recorded_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "context" + "encoding/json" + "path/filepath" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" + + "github.com/novox/mesh-host/internal/liveness" + "github.com/novox/mesh-host/internal/store" +) + +// The record check reads the engine's applied state: a composition one of whose resources has no record there is not +// recorded (issue 1945). +func TestACompositionIsRecordedOnlyWhenEveryResourceIs(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + var s store.State + s.Record(store.Applied{ID: "ca-trust.trust", Type: "file", Target: "/etc/ca"}) + if err := store.Save(path, s); err != nil { + t.Fatal(err) + } + c := delivery.Composed{Node: "ace", Module: "ca-trust", Resources: []json.RawMessage{ + json.RawMessage(`{"id":"ca-trust.trust","type":"file","module":"ca-trust","path":"/etc/ca","mode":"0644","content":"x"}`)}} + recorded := recordedIn(path) + if !recorded(context.Background(), c) { + t.Fatal("a composition whose every resource has a record was not recorded") + } + c.Resources = append(c.Resources, json.RawMessage( + `{"id":"ca-trust.more","type":"file","module":"ca-trust","path":"/etc/ca2","mode":"0644","content":"y"}`)) + if recorded(context.Background(), c) { + t.Fatal("a composition with a resource the state holds no record of was recorded") + } +} + +// The modules applied again are those with a missing resource, each once; one down but there is not among them. +func TestTheModulesAppliedAgainAreThoseWithAMissingResource(t *testing.T) { + st := liveness.Statement{Resources: []liveness.State{ + {Resource: liveness.Resource{Module: "matrix", ID: "matrix.homeserver"}, State: liveness.Unhealthy, Missing: true}, + {Resource: liveness.Resource{Module: "matrix", ID: "matrix.db"}, State: liveness.Unhealthy, Missing: true}, + {Resource: liveness.Resource{Module: "ca-trust", ID: "ca-trust.trust"}, State: liveness.Unhealthy, Missing: true}, + {Resource: liveness.Resource{Module: "web", ID: "web.server"}, State: liveness.Unhealthy}, + }} + if got := missingModules(st); len(got) != 2 || got[0] != "ca-trust" || got[1] != "matrix" { + t.Fatalf("applied again: %v", got) + } +} diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 40f6fe4..b5350e8 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -94,7 +94,10 @@ type Env struct { // composition unsigned, signed by another key, or changed after it was signed is refused and nothing of it is // applied. Nil trusts no key, so nothing is applied. Trusted func() []ed25519.PublicKey - Say func(string) + // Recorded says the engine holds a record of every resource of the composition, so it was applied and not removed + // since; nil says yes. A composition the kept declaration or a have says runs is applied again when it says no. + Recorded func(ctx context.Context, c delivery.Composed) bool + Say func(string) } // Agent follows the wants. @@ -563,6 +566,29 @@ func (a *Agent) settleOne(ctx context.Context, module string) { // own have (what this node-engine last applied), else the declaration it kept before the switch (ADR 0539 §9). True when // it is, with the have written; nothing is applied. func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp compNow) bool { + // What a have or the kept declaration says runs is believed only when the engine still holds a record of each of its + // resources: a resource removed since (a node's own apply took it, issue 1945) is made again by an apply. + if a.env.Recorded != nil && !a.env.Recorded(ctx, comp.C) { + if module != NodeModule { + if _, ok := a.env.Haves.Read(ctx, module); ok { + a.mu.Lock() + a.prior[module] = true + a.mu.Unlock() + } + } + if a.env.Kept != nil { + if _, ok := a.env.Kept()[module]; ok { + a.mu.Lock() + a.prior[module] = true + a.mu.Unlock() + } + } + a.mu.Lock() + a.sayOnce("unrecorded:"+module+":"+comp.Dig, fmt.Sprintf("%s's composition %s is applied again: this engine holds "+ + "no record of some of its resources", module, short(comp.Dig))) + a.mu.Unlock() + return false + } if module != NodeModule { if h, ok := a.env.Haves.Read(ctx, module); ok { a.mu.Lock() @@ -718,3 +744,13 @@ func Canonical(raw json.RawMessage) (json.RawMessage, error) { } return json.Marshal(v) } + +func short(s string) string { + if i := strings.IndexByte(s, ':'); i >= 0 { + s = s[i+1:] + } + if len(s) > 12 { + s = s[:12] + } + return s +} diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 793d9a3..320cb19 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -782,3 +782,30 @@ func TestAStaleRefusedCompositionDoesNotHoldTheNodesOwnBack(t *testing.T) { defer stop() eventually(t, "the node's own not applied", func() bool { return len(b.appliedList()) == 1 }) } + +// A composition the kept declaration or the node's own have says runs is applied all the same when the engine holds no +// record of its resources: they were removed since (issue 1945, live on 2026-10-25: ace's and g14's node-engines removed +// ca-trust's, power's and matrix's resources in a node's own apply, and the wants that named those modules' own +// compositions were judged already running from the kept declaration, so nothing made them again). +func TestACompositionWhoseResourcesWereRemovedIsAppliedAgain(t *testing.T) { + for _, from := range []string{"kept", "have"} { + t.Run(from, func(t *testing.T) { + b := newBed(t) + fp := fpOf("a") + rs := []json.RawMessage{file("ca-trust", "trust", "1")} + ck, craw, dig := composition(t, "ca-trust", fp, false, rs...) + wk, wraw := wantFor(t, "ca-trust", fp, dig, "") + b.comps.before(ck, craw) + b.wants.before(wk, wraw) + if from == "kept" { + b.kept = map[string][]json.RawMessage{"ca-trust": rs} + } else { + b.haves.stored = map[string]Have{"ca-trust": {Module: "ca-trust", FP: fp, Composed: dig, Phase: "running"}} + } + b.env.Recorded = func(context.Context, delivery.Composed) bool { return false } + stop := b.run() + defer stop() + eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 }) + }) + } +} diff --git a/internal/liveness/liveness.go b/internal/liveness/liveness.go index 2ae6c9e..9491471 100644 --- a/internal/liveness/liveness.go +++ b/internal/liveness/liveness.go @@ -172,6 +172,9 @@ type State struct { Restarts int `json:"restarts,omitempty"` // Waits is what a resource stated Waiting waits for (ADR 0283). Waits []health.Wait `json:"waits,omitempty"` + // Missing is a resource the runtime or the service manager does not have at all after its grace: removed, not + // stopped. The node agent applies its module again (issue 1945). + Missing bool `json:"missing,omitempty"` } // CheckOf and NeedsOf are a stated resource's declared check, in a word, and the provision it exercises. @@ -358,7 +361,8 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) { j.judge(k, o, now, r.Kind == KindContainer && held[r.Target], blind, why) seen[r.ID] = true st.Resources = append(st.Resources, State{Resource: k.Resource, State: k.State, Reason: k.Reason, Since: k.Since, - Streak: k.Streak, Restarts: k.Counted, Waits: waitsOf(k)}) + Streak: k.Streak, Restarts: k.Counted, Waits: waitsOf(k), + Missing: k.State == Unhealthy && k.Reason == ReasonDown && !blind && !o.Found}) // What it waits for is part of what is said: a change of waits alone is a change. word := k.State + "/" + k.Reason + "/" + waitsWord(waitsOf(k)) if j.said[r.ID] != word { diff --git a/internal/liveness/liveness_test.go b/internal/liveness/liveness_test.go index 589dd16..74b4764 100644 --- a/internal/liveness/liveness_test.go +++ b/internal/liveness/liveness_test.go @@ -383,3 +383,28 @@ func TestAUserUnitsManagerNotRunningIsNotReadAndNotDown(t *testing.T) { t.Fatalf("the running manager is read in the account's own: %q", last) } } + +// A resource the runtime does not have at all after its grace is said missing, beside down: what the node agent applies +// again (issue 1945). One that is there and stopped is down and not missing, and one that cannot be read is neither. +func TestAResourceNotThereIsMissingAndOneStoppedIsNot(t *testing.T) { + c := &clock{now: t0} + rt := &fakeRuntime{containers: map[string]Observed{}} + j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName)) + j.Set([]Resource{server}) + j.Look(t.Context()) + c.now = t0.Add(2 * time.Minute) + st, _ := j.Look(t.Context()) + if s := stateOf(t, st, "letta.server"); !s.Missing { + t.Fatalf("a container not there after its grace is not missing: %+v", s) + } + rt.containers["letta-server"] = Observed{Found: true, Identity: "c1"} + st, _ = j.Look(t.Context()) + if s := stateOf(t, st, "letta.server"); s.Missing { + t.Fatalf("a container there and stopped: %+v", s) + } + rt.err = errors.New("cannot connect") + st, _ = j.Look(t.Context()) + if s := stateOf(t, st, "letta.server"); s.Missing { + t.Fatalf("a runtime that does not answer says nothing is missing: %+v", s) + } +} -- 2.54.0 From 43d1c2456fb22cbdb54a5ffdf07b2bd158f8cb29 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 25 Oct 2026 11:04:08 +0100 Subject: [PATCH 2/3] Apply a module again only for a newly missing resource, leave filled files out of the record check, and say when the applied state cannot be read (issue 1945) --- cmd/mesh-host/main.go | 23 +++++++++++++++- cmd/mesh-host/nodeagent.go | 15 +++++++++-- cmd/mesh-host/recorded_test.go | 48 +++++++++++++++++++++++++++++++++- 3 files changed, 82 insertions(+), 4 deletions(-) diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 01ae6ba..7b13709 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1334,6 +1334,7 @@ const ( // anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240 // rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it. func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce, reapply func(modules ...string)) { + var missingBefore map[string]bool ticker := time.NewTicker(liveness.LookEvery) defer ticker.Stop() var lastSaid time.Time @@ -1421,7 +1422,9 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, } // A resource the machine does not have at all is made again: its module's composition applied anew (issue // 1945), once each time the statement changes, never on a clock. - if m := missingModules(st); len(m) > 0 && reapply != nil { + var m []string + m, missingBefore = newlyMissing(missingBefore, st) + if len(m) > 0 && reapply != nil { say(fmt.Sprintf("applying %s again: a resource it declares is not on this machine", strings.Join(m, ", "))) reapply(m...) } @@ -2101,3 +2104,21 @@ func missingModules(st liveness.Statement) []string { sort.Strings(out) return out } + +// newlyMissing are the modules of the resources a statement says missing that the last statement acted on did not, each +// once, sorted, and the missing resources now: a resource still missing is not acted on again, so a module whose +// resource cannot be made is not applied at every change of the statement (the review of issue 1945). +func newlyMissing(last map[string]bool, st liveness.Statement) ([]string, map[string]bool) { + now := map[string]bool{} + var fresh liveness.Statement + for _, r := range st.Resources { + if !r.Missing { + continue + } + now[r.ID] = true + if !last[r.ID] { + fresh.Resources = append(fresh.Resources, r) + } + } + return missingModules(fresh), now +} diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index e4e902e..7dd18f9 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -293,7 +293,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess Composed: bucketWatch{session, composedBucket}, Haves: sessionHaves{session}, Kept: keptByModule(opts, signer), - Recorded: recordedIn(opts.state), + Recorded: recordedIn(opts.state, say), Trusted: trustedKeys(opts, signer), Backup: backupNode(session), Say: say, @@ -519,17 +519,28 @@ func refuseEmptyWrites(d *declaration.Declaration) error { // recordedIn says the engine's applied state holds a record of every resource of a composition: what the agent believes // of a have or the kept declaration only when it does, so a resource removed since is made again (issue 1945). A state // that cannot be read says yes: the agent then believes what it was told, as before. -func recordedIn(statePath string) func(ctx context.Context, c delivery.Composed) bool { +func recordedIn(statePath string, say func(string)) func(ctx context.Context, c delivery.Composed) bool { return func(_ context.Context, c delivery.Composed) bool { known, err := store.Load(statePath) if err != nil { + if say != nil { + say(fmt.Sprintf("the applied state cannot be read, so what runs is believed as told: %v", err)) + } return true } + filled := make(map[string]bool, len(c.Filled)) + for _, p := range c.Filled { + filled[p] = true + } for _, raw := range c.Resources { r, err := declaration.ParseResource(raw) if err != nil { return true } + // A file the provisioner client fills is left to it, and has no record until it writes it. + if f, ok := r.(*declaration.File); ok && filled[f.Path] { + continue + } if _, ok := known.Find(r.Identity()); !ok { return false } diff --git a/cmd/mesh-host/recorded_test.go b/cmd/mesh-host/recorded_test.go index bba28e2..ce7684b 100644 --- a/cmd/mesh-host/recorded_test.go +++ b/cmd/mesh-host/recorded_test.go @@ -23,7 +23,7 @@ func TestACompositionIsRecordedOnlyWhenEveryResourceIs(t *testing.T) { } c := delivery.Composed{Node: "ace", Module: "ca-trust", Resources: []json.RawMessage{ json.RawMessage(`{"id":"ca-trust.trust","type":"file","module":"ca-trust","path":"/etc/ca","mode":"0644","content":"x"}`)}} - recorded := recordedIn(path) + recorded := recordedIn(path, func(string) {}) if !recorded(context.Background(), c) { t.Fatal("a composition whose every resource has a record was not recorded") } @@ -46,3 +46,49 @@ func TestTheModulesAppliedAgainAreThoseWithAMissingResource(t *testing.T) { t.Fatalf("applied again: %v", got) } } + +// A module is applied again only for a resource newly missing: one still missing at the next change of the statement is +// not acted on again, so a resource that cannot be made is not applied (and backed up) at every unrelated change. One +// that comes back and goes again is newly missing again (the review of issue 1945). +func TestOnlyANewlyMissingResourceAppliesItsModuleAgain(t *testing.T) { + missing := func(ids ...string) liveness.Statement { + var st liveness.Statement + for _, id := range ids { + st.Resources = append(st.Resources, liveness.State{Resource: liveness.Resource{Module: "matrix", ID: id}, + State: liveness.Unhealthy, Missing: true}) + } + return st + } + var last map[string]bool + var got []string + got, last = newlyMissing(last, missing("matrix.homeserver")) + if len(got) != 1 { + t.Fatalf("first missing: %v", got) + } + got, last = newlyMissing(last, missing("matrix.homeserver")) + if len(got) != 0 { + t.Fatalf("still missing was acted on again: %v", got) + } + got, last = newlyMissing(last, missing()) + got, last = newlyMissing(last, missing("matrix.homeserver")) + if len(got) != 1 { + t.Fatalf("missing again after it came back: %v", got) + } +} + +// A file the provisioner client fills is left to it by the apply and has no record until the client writes it: it does +// not make its composition unrecorded (the review of issue 1945). +func TestAFilledFileDoesNotMakeItsCompositionUnrecorded(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + var s store.State + s.Record(store.Applied{ID: "ca-trust.trust", Type: "file", Target: "/etc/ca"}) + if err := store.Save(path, s); err != nil { + t.Fatal(err) + } + c := delivery.Composed{Node: "ace", Module: "ca-trust", Filled: []string{"/etc/ca-filled"}, Resources: []json.RawMessage{ + json.RawMessage(`{"id":"ca-trust.trust","type":"file","module":"ca-trust","path":"/etc/ca","mode":"0644","content":"x"}`), + json.RawMessage(`{"id":"ca-trust.filled","type":"file","module":"ca-trust","path":"/etc/ca-filled","mode":"0644","owner-at-runtime":"owned"}`)}} + if !recordedIn(path, func(string) {})(context.Background(), c) { + t.Fatal("a filled file the client has not written made its composition unrecorded") + } +} -- 2.54.0 From 958ccbde3ed4b5c9b43b2a75db8b30e5165157e1 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 25 Oct 2026 11:08:10 +0100 Subject: [PATCH 3/3] Say a running module ok once looked at when nothing of it is judged, so mesh-host's canary can be healthy and its other nodes get the new engine (issue 1945) --- internal/link/have.go | 28 +++++++++++++++++++++------- internal/link/have_test.go | 23 +++++++++++++++++++++++ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/internal/link/have.go b/internal/link/have.go index caa01c1..35ec1c0 100644 --- a/internal/link/have.go +++ b/internal/link/have.go @@ -36,7 +36,10 @@ type haveState struct { mu sync.Mutex modules map[string]ModuleFact health map[string]delivery.Health - node *delivery.HaveNode + // looked says a look's health was taken: a running module it judges no resource of then has nothing that could be + // down, and is ok (issue 1945). + looked bool + node *delivery.HaveNode // nodeComposed is the digest of the node's own composition it runs (ADR 0539 §9), kept beside the node's facts. nodeComposed string // written is the build of each module this node-engine wrote a have key for and that the declaration it applied @@ -110,7 +113,7 @@ func (q *Session) noteApply(r Report) bool { s.mu.Lock() defer s.mu.Unlock() if r.Health != nil { - s.health = moduleHealths(r.Health) + s.health, s.looked = moduleHealths(r.Health), true } n := nodeFacts(r) s.node = &n @@ -152,7 +155,7 @@ func (q *Session) noteApply(r Report) bool { } s.modules = next if r.Health != nil { - s.health = moduleHealths(r.Health) + s.health, s.looked = moduleHealths(r.Health), true } n := nodeFacts(r) s.node = &n @@ -166,20 +169,20 @@ func (q *Session) noteHealth(h Health) bool { defer s.mu.Unlock() next := moduleHealths(&h) changed := false - for module := range s.modules { + for module, f := range s.modules { want, ok := next[module] if !ok { - want = delivery.HealthUnknown + want = nothingJudged(f, true) } have, known := s.health[module] if !known { - have = delivery.HealthUnknown + have = nothingJudged(f, s.looked) } if have != want { changed = true } } - s.health = next + s.health, s.looked = next, true return changed } @@ -232,6 +235,8 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) { } if got, known := s.health[module]; known { h.Health = got + } else { + h.Health = nothingJudged(f, s.looked) } key, err := delivery.HaveKey(node, module) if err != nil { @@ -452,3 +457,12 @@ func (q *Session) ReadHave(ctx context.Context, module string) (delivery.Have, b } return h, true } + +// nothingJudged is the health of a module a look judged no resource of: ok when it was looked at and the module applied +// and runs (nothing of it could be down), unknown otherwise. +func nothingJudged(f ModuleFact, looked bool) delivery.Health { + if looked && !f.Failed && (f.Phase == "" || f.Phase == string(delivery.PhaseRunning)) { + return delivery.HealthOK + } + return delivery.HealthUnknown +} diff --git a/internal/link/have_test.go b/internal/link/have_test.go index 5402d54..a16b840 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -498,3 +498,26 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth( t.Fatalf("the node key: %+v %v", n, err) } } + +// A module applied and running that has nothing long-running to judge is ok once the engine has looked: there is nothing +// of it that could be down. Unknown kept it from ever being healthy, so the assigner's canary rule held every other node +// of mesh-host back for good (issue 1945, live on 2026-10-25: ace's have of mesh-host said unknown, and novox and shanks +// never got the new node-engine). A failed one stays failed, and before any look it is unknown. +func TestAModuleWithNothingToJudgeIsOkOnceLookedAt(t *testing.T) { + l := newHaveLink() + q, _ := aHaveQueue(l) + q.SayFacts(context.Background(), Report{Modules: []ModuleFact{{Module: "mesh-host", Build: "b"}, {Module: "web", Build: "b"}, + {Module: "broken", Build: "b", Failed: true, Detail: "x"}}}) + if h := l.have(t, "n1.mesh-host").Health; h != delivery.HealthUnknown { + t.Fatalf("before any look: %s", h) + } + if !q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateHealthy})) { + t.Fatal("not said") + } + if h := l.have(t, "n1.mesh-host").Health; h != delivery.HealthOK { + t.Fatalf("a running module with nothing to judge after a look: %s", h) + } + if h := l.have(t, "n1.broken").Health; h == delivery.HealthOK { + t.Fatalf("a failed module was said ok") + } +} -- 2.54.0