From 744bfec4a018936d0c00e6ad25d85a7ac2550a9e Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 30 Oct 2026 18:12:56 +0100 Subject: [PATCH] Hand the health loop's judges what runs once a module settles, so a resource its new composition drops is no longer watched (issue 774) restic's composition dropped its process restic.measure on g14 and shanks at 16:14; the engine removed it and said it down on every look after. The judges were set only when facts were said, and the facts said inside an apply read the module's composition from before it (the agent records the new one only after the apply returns), so the removed resource was put back. Judge.Set itself prunes what is not declared. Quick fix: set the judges after the restic apply. Thorough fix (chosen): one function hands every judge (liveness, resolver, units, accounts, libraries) what is declared; it runs from the facts as before and now also whenever a module settles (applied or removed) or is taken as running, so every removed health entry of every module is pruned at recompose. --- cmd/mesh-host/main.go | 46 ++++++++++++++++++++++++++++++---- cmd/mesh-host/nodeagent.go | 12 ++++++--- cmd/mesh-host/whatruns_test.go | 43 +++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 8 deletions(-) diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 505bc93..c689a92 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1840,9 +1840,8 @@ func factsOf(ctx context.Context, opts options, declared *declaration.Declaratio if j := judging; j != nil { // A container that waits for a library whose disk or mount is not there was not created: the library says // why, and liveness judges it neither alive nor dead (novox/hq ADR 0328 §9). - j.Set(liveness.LongRunning(declared, withWaiting(held, apply.WaitingOnLibrary(declared)))) + judgeWhatIsDeclared(declared, held) st, _ := j.Look(ctx) - netJudge.declare(declaredResolvConf(declared)) var netSt *network.Statement if n := netJudge.get(); n != nil { ns := n.Last() @@ -1850,7 +1849,6 @@ func factsOf(ctx context.Context, opts options, declared *declaration.Declaratio } var unitSt *units.Statement if u := unitJudge; u != nil { - u.Set(units.OwnedBy(declared, held)) us := u.Last() unitSt = &us } @@ -1858,11 +1856,9 @@ func factsOf(ctx context.Context, opts options, declared *declaration.Declaratio // said with it, relogin needed included, in the report that says the apply. var accountSt *accounts.Statement if a := accountJudge; a != nil { - a.Set(accounts.Of(declared, held)) as, _ := a.Look(ctx) accountSt = &as } - accesses.set(declared, held) // A take act stopped at its bound goes on, beside (ADR 0328 §7). keeping.poke(ctx) accessSt, _ := accesses.look(time.Now()) @@ -1957,6 +1953,46 @@ func factsOf(ctx context.Context, opts options, declared *declaration.Declaratio return report } +// judgeWhatIsDeclared hands every judge of the health loop what is declared here now: the long-running resources, the +// resolver file, the units, the accounts and the libraries. A resource, unit, account or library no longer declared is +// forgotten by its judge, so a module whose composition drops one is no longer watched for it. +func judgeWhatIsDeclared(declared *declaration.Declaration, held map[string]bool) { + if j := judging; j != nil { + // A container that waits for a library whose disk or mount is not there was not created: the library says + // why, and liveness judges it neither alive nor dead (novox/hq ADR 0328 §9). + j.Set(liveness.LongRunning(declared, withWaiting(held, apply.WaitingOnLibrary(declared)))) + } + netJudge.declare(declaredResolvConf(declared)) + if u := unitJudge; u != nil { + u.Set(units.OwnedBy(declared, held)) + } + if a := accountJudge; a != nil { + a.Set(accounts.Of(declared, held)) + } + accesses.set(declared, held) +} + +// judgeWhatRunsNow hands the judges what runs now, as the node agent holds it once a module settled (applied or +// removed) or was taken as running: the facts said inside an apply read the module's composition from before it, so a +// resource its new composition dropped stayed watched until the next facts (g14 and shanks, 2026-10-30 16:14: restic's +// process restic.measure, removed, then said down every look). +func judgeWhatRunsNow(statePath string, signer ed25519.PublicKey) { + if judging == nil { + return + } + declared, err := declaredNow(statePath, signer) + if err != nil { + return + } + held := map[string]bool{} + if known, err := store.Load(statePath); err == nil { + for _, h := range known.Held { + held[h.ID] = true + } + } + judgeWhatIsDeclared(declared, held) +} + // sayFacts says this node's facts on the link open now (novox/hq ADR 0539 §9): what the node is made of, how what runs // here is, and its filters and ports. What runs is the compositions the node agent holds running (ag), or, before it // has run, the declaration the node kept before the switch. It applies nothing; the haves are the agent's. diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 7d3ba5f..9e55e69 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -555,11 +555,17 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus Backup: backupNode(session), // What the filling refused before this apply may be named now: the provisioner's client is told to fill // again (ace, 2026-10-25 21:17). - Settled: func(module string) { knockFilling(module+" settled", say) }, + Settled: func(module string) { + knockFilling(module+" settled", say) + judgeWhatRunsNow(opts.state, signer) + }, // And once the haves are read at the start, and whenever a module is taken as running without an apply, so // a refusal from the engine's start does not wait for some module to settle (issue 924). - RunsChanged: func(why string) { knockFilling(why, say) }, - Say: say, + RunsChanged: func(why string) { + knockFilling(why, say) + judgeWhatRunsNow(opts.state, signer) + }, + Say: say, // A failure of the bus not answering for a moment is tried again: 2 s, 5 s, 15 s, 60 s, then every 5 min. Backoff: link.Backoff, Apply: func(ctx context.Context, c delivery.Composed, others []delivery.Composed) agent.Result { diff --git a/cmd/mesh-host/whatruns_test.go b/cmd/mesh-host/whatruns_test.go index 1d4601e..bdace6b 100644 --- a/cmd/mesh-host/whatruns_test.go +++ b/cmd/mesh-host/whatruns_test.go @@ -1,10 +1,12 @@ package main import ( + "context" "crypto/ed25519" "encoding/json" "errors" "path/filepath" + "sort" "strings" "testing" @@ -12,6 +14,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-host/internal/agent" + "github.com/novox/mesh-host/internal/liveness" "github.com/novox/mesh-host/internal/store" ) @@ -62,6 +65,46 @@ func TestDeclaredNowReadsWhatRunsWhenTheAgentIsRunning(t *testing.T) { t.Fatalf("declaredNow read %+v (%v), not what runs", d, err) } + // And the health loop's judges are handed what runs once a module settles (g14 and shanks, 2026-10-30 16:14): + // restic's composition dropped its process restic.measure, which stayed watched and said down on every look. + j, err := liveness.Open(filepath.Join(t.TempDir(), "liveness.json"), fakeRuntime{}) + if err != nil { + t.Fatal(err) + } + judging = j + defer func() { judging = nil }() + process := func(name string) json.RawMessage { + raw, _ := json.Marshal(map[string]any{"id": "restic." + name, "type": "process", "name": "restic-" + name, + "source": "https://store.invalid/restic", "digest": "sha256:" + strings.Repeat("a", 64), + "run": []string{"./restic"}, "module": "restic"}) + return raw + } + for _, row := range []struct { + runs []string + watch string + }{{[]string{"night", "measure"}, "restic.measure restic.night"}, {[]string{"night"}, "restic.night"}} { + var raws []json.RawMessage + for _, name := range row.runs { + raws = append(raws, process(name)) + } + runs, err := whatRuns([]delivery.Composed{{Node: "n1", Module: "restic", Resources: raws}}, false, + func() (*declaration.Declaration, error) { return nil, errors.New("none kept") }) + if err != nil { + t.Fatal(err) + } + setRunsHere(func(string, ed25519.PublicKey) (*declaration.Declaration, error) { return runs, nil }) + judgeWhatRunsNow(filepath.Join(t.TempDir(), "state.json"), nil) + st, _ := j.Look(context.Background()) + var watched []string + for _, r := range st.Resources { + watched = append(watched, r.ID) + } + sort.Strings(watched) + if got := strings.Join(watched, " "); got != row.watch { + t.Errorf("running %v, the judge watches %q, want %q", row.runs, got, row.watch) + } + } + // L3 s9 (ADR 0550 §2): a node whose placed modules all have a have reads the kept declaration no more, though the // file is there and readable and the controller's key is dropped; until then it reads it. state := filepath.Join(t.TempDir(), "state", "state.json")