From d8de0a8853a214526b16a0f08f1de1e4ab30ef0f Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 01:26:46 +0100 Subject: [PATCH] Judge a module the agent has not taken yet by the composition its have names, and knock the filling at the start and whenever a module is taken as running (issue 924) A held module on a node composing its own never gets a composition, so the agent never took it and the filling judged its files by the kept declaration, older than power's three new drop-ins: refused from every engine start on ace, g14 and shanks until some module settled (2026-10-29 00:09 UTC). The second meeting of issue 924. Quick fix: knock the filling once the haves are read at the start. Thorough fix (chosen, the kind recurs for every file added since the switch): what runs of such a module is the composition its have names, from the bucket or its revisions, ahead of the kept declaration; the filling is knocked on every change of what runs, not only on a settle. One unit test replaced by a table of four cases; the count stays. --- cmd/mesh-host/filling.go | 11 ++-- cmd/mesh-host/nodeagent.go | 7 +- internal/agent/agent.go | 123 +++++++++++++++++++++++++++++++++-- internal/agent/agent_test.go | 91 ++++++++++++++++++++++---- 4 files changed, 208 insertions(+), 24 deletions(-) diff --git a/cmd/mesh-host/filling.go b/cmd/mesh-host/filling.go index d77d2e9..5257e3e 100644 --- a/cmd/mesh-host/filling.go +++ b/cmd/mesh-host/filling.go @@ -30,9 +30,10 @@ func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, s var fillingNow atomic.Pointer[filling.Engine] // knockFilling tells the provisioner's client to fill again what the filling refused or failed and the declaration -// applied now may let through (filling.Engine.Knock), once a module's apply or removal settled (agent.Env.Settled). -// What it told, or why it could not, is said. -func knockFilling(module string, say func(string)) { +// applied now may let through (filling.Engine.Knock), after what runs changed: a module's apply or removal settled +// (agent.Env.Settled, " settled"), the haves were read at the start, or a module was taken as running +// (agent.Env.RunsChanged, issue 924). What it told, or why it could not, is said. +func knockFilling(after string, say func(string)) { e := fillingNow.Load() if e == nil { return @@ -40,9 +41,9 @@ func knockFilling(module string, say func(string)) { told, err := e.Knock(filling.Socket) switch { case err != nil: - say(fmt.Sprintf("after %s settled: %v", module, err)) + say(fmt.Sprintf("after %s: %v", after, err)) case told != "": - say(fmt.Sprintf("after %s settled, the provisioner's client is told to fill again: %s", module, told)) + say(fmt.Sprintf("after %s, the provisioner's client is told to fill again: %s", after, told)) } } diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 667d18e..781045d 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -510,8 +510,11 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus Backup: backupNode(session), // What the filling refused before this apply may be named now: the provisioner's client is told to fill // again (ace, 2026-10-25 21:17). - Settled: func(module string) { knockFilling(module, say) }, - Say: say, + Settled: func(module string) { knockFilling(module+" settled", say) }, + // And once the haves are read at the start, and whenever a module is taken as running without an apply, so + // a refusal from the engine's start does not wait for some module to settle (issue 924). + RunsChanged: func(why string) { knockFilling(why, say) }, + Say: say, // A failure of the bus not answering for a moment is tried again: 2 s, 5 s, 15 s, 60 s, then every 5 min. Backoff: link.Backoff, Apply: func(ctx context.Context, c delivery.Composed, others []delivery.Composed) agent.Result { diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 5bf7222..62df761 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -109,7 +109,9 @@ type Env struct { Kept func() map[string][]json.RawMessage // KeptFilled is the paths the kept declaration names as filled by the provisioner, by the module whose file is at // each (the node's own under NodeModule). A module that still runs from the kept declaration keeps its filled files - // filled: without them the filling refused each one as "not named as filled" (issue 1945). Nil names none. + // filled: without them the filling refused each one as "not named as filled" (issue 1945). Nil names none. A module + // whose have names a composition the bucket keeps runs from that composition, never from the kept declaration + // (issue 924): the kept declaration is older than every composition applied since the switch. KeptFilled func() map[string][]string // Trusted is the public keys a composition may be signed by: the controller's and the operator's (ADR 0539 §3). A // composition unsigned, signed by another key, or changed after it was signed is refused and nothing of it is @@ -131,6 +133,12 @@ type Env struct { // them. The node-engine's filling tells the provisioner's client then to fill again what it refused (mesh-host // internal/filling Knock; ace, 2026-10-25 21:17). Nil tells nothing. Settled func(module string) + // RunsChanged is told, outside the agent's lock, that what runs (Everything) changed other than by an apply or a + // removal: once at the start, when the haves and the compositions they name are read, and each time a module is + // taken as running from its have or the kept declaration. Like Settled, the filling then tells the provisioner's + // client to fill again what it refused (issue 924, the second meeting: ace, g14 and shanks on 2026-10-29 at 00:09 + // UTC refused power's three new drop-ins from an engine start on, as no module settled after it). Nil tells nothing. + RunsChanged func(why string) // Backoff is how long to wait before the try-th retry (1, 2, …) of an apply that failed transiently. Nil retries a // transient failure only on RetryTransient or a change of either key. Backoff func(try int) time.Duration @@ -160,6 +168,10 @@ type Agent struct { // applied is the composition each module was last applied from: a revision a want named from the key's history is // not the key's latest value, and what it holds is still held out of the others' applies. applied map[string]compNow + // had is, for each module whose have names a composition the bucket keeps (its latest value or a revision), that + // composition, read once at the start (loadHad): what runs of a module the agent has not taken yet, ahead of the kept + // declaration (issue 924). + had map[string]compNow // attempt is the last apply of a module that failed or is held, with the revisions it was made from. attempt map[string]attempt checked map[string]bool @@ -218,7 +230,7 @@ func New(env Env) *Agent { if env.Say == nil { env.Say = func(string) {} } - return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{}, + return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{}, had: map[string]compNow{}, attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, notComposed: map[string]notComposedMark{}, retry: make(chan []string, 8), reconcile: make(chan []string, 64), tries: map[string]int{}, again: make(chan againAsk, 64), transient: make(chan struct{}, 1), stopped: make(chan struct{})} @@ -236,7 +248,7 @@ func (a *Agent) Run(ctx context.Context) error { if err != nil { return fmt.Errorf("the agent cannot watch its compositions: %w", err) } - var wantsCaught, compsCaught bool + var wantsCaught, compsCaught, hadRead bool dirty := map[string]bool{} for { select { @@ -303,7 +315,17 @@ func (a *Agent) Run(ctx context.Context) error { } // Nothing is applied until what was there at the start is all in: a module's composition may arrive after its // want, and the node's own composition comes before any module's. - if !(wantsCaught && compsCaught) || len(dirty) == 0 { + if !(wantsCaught && compsCaught) { + continue + } + if !hadRead { + // Before the first settle: what runs of each module with a have, from the composition it names, so the + // filling judges by it at once and not by the kept declaration until some module settles (issue 924). + hadRead = true + a.loadHad(ctx) + a.runsChanged("the haves were read at the start") + } + if len(dirty) == 0 { continue } a.settle(ctx, dirty) @@ -726,7 +748,12 @@ func (a *Agent) revisionNamed(ctx context.Context, module, fp, digest string) (c if err != nil { continue } - dig, err := delivery.VerifyComposed(c, trusted) + var dig string + if a.env.SelfComposed { + dig, err = delivery.ComposedDigestOf(c) // the node's own: unsigned (ADR 0547 §2) + } else { + dig, err = delivery.VerifyComposed(c, trusted) + } if err != nil || dig != digest { continue } @@ -755,13 +782,27 @@ func (a *Agent) others(except string) []delivery.Composed { out = append(out, c.C) } else if c, ok := a.applied[m]; ok && c.Dig == h.Composed { out = append(out, c.C) + } else if c, ok := a.had[m]; ok && c.Dig == h.Composed { + out = append(out, c.C) // a revision its have named, replaced since under the same key } } + hadNames := make([]string, 0, len(a.had)) + for m := range a.had { + if _, held := a.running[m]; !held && m != except { + hadNames = append(hadNames, m) + } + } + sort.Strings(hadNames) + for _, m := range hadNames { + out = append(out, a.had[m].C) + } if a.env.Kept != nil { kept := a.env.Kept() names := make([]string, 0, len(kept)) for m := range kept { - if _, held := a.running[m]; !held && m != except { + _, held := a.running[m] + _, had := a.had[m] + if !held && !had && m != except { names = append(names, m) } } @@ -780,6 +821,73 @@ func (a *Agent) others(except string) []delivery.Composed { return out } +// loadHad reads, once at the start, the have of every module with a want or in the kept declaration and, where it names +// a composition, that composition: the bucket's value at its key when its digest is the have's, else the revision the +// bucket keeps with that digest. A have naming none, or one the bucket no longer keeps, leaves the module to the kept +// declaration, said once (issue 924). +func (a *Agent) loadHad(ctx context.Context) { + a.mu.Lock() + seen := map[string]bool{} + for m, w := range a.wants { + if m != NodeModule && w.Present && !w.Want.Removed { + seen[m] = true + } + } + if a.env.Kept != nil { + for m := range a.env.Kept() { + if m != NodeModule { + seen[m] = true + } + } + } + a.mu.Unlock() + names := make([]string, 0, len(seen)) + for m := range seen { + names = append(names, m) + } + sort.Strings(names) + for _, m := range names { + if ctx.Err() != nil { + return + } + h, ok := a.env.Haves.Read(ctx, m) + if !ok || h.Composed == "" || h.Phase == string(delivery.PhaseRemoved) { + continue + } + a.mu.Lock() + c, found := a.compositionFor(m, h.FP) + a.mu.Unlock() + if !found || c.Err != nil || c.Dig != h.Composed { + c, found = a.revisionNamed(ctx, m, h.FP, h.Composed) + } + a.mu.Lock() + if found { + a.had[m] = c + } else if _, kept := a.keptHas(m); kept { + a.sayOnce("had:"+m, fmt.Sprintf("%s's have names the composition %s, which the bucket no longer keeps: it runs "+ + "from the kept declaration until it is applied or taken", m, short(h.Composed))) + } + a.mu.Unlock() + } +} + +// keptHas says whether the kept declaration has resources for the module. Called holding a.mu or not: Kept takes no lock +// of the agent's. +func (a *Agent) keptHas(module string) ([]json.RawMessage, bool) { + if a.env.Kept == nil { + return nil, false + } + rs, ok := a.env.Kept()[module] + return rs, ok +} + +// runsChanged tells RunsChanged, outside the agent's lock. +func (a *Agent) runsChanged(why string) { + if a.env.RunsChanged != nil { + a.env.RunsChanged(why) + } +} + // unreadableModules are the keys of the module compositions in the node's bucket that cannot be read or are refused, of a // module the assigner has not removed: for each module only the composition its want names, else its newest, since a // stale one nobody applies (written before a rule the SDK now holds) must not hold the node's own back. Called holding @@ -1048,6 +1156,7 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning} } a.mu.Unlock() + a.runsChanged(module + " was taken as held") return true } if h.Composed != "" { @@ -1068,6 +1177,7 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com a.mu.Lock() a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning} a.mu.Unlock() + a.runsChanged(module + " was taken from its have") // Said again, so the session holds the module and its key carries the health this engine judges, // not the one an older engine left there (xorg's unknown on g14 and shanks, 2026-10-25). a.writeHave(ctx, Have{Module: module, FP: h.FP, Composed: h.Composed, @@ -1096,6 +1206,7 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com a.mu.Lock() a.running[module] = delivery.Have{FP: w.Want.FP, Composed: w.Want.Composed, Phase: delivery.PhaseRunning} a.mu.Unlock() + a.runsChanged(module + " was taken from the kept declaration") a.writeHave(ctx, Have{Module: module, FP: w.Want.FP, Composed: w.Want.Composed, Phase: string(delivery.PhaseRunning)}) return true } diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 7c256bb..f3c0b7e 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -1026,18 +1026,87 @@ func TestTheRevisionAWantNamesIsAppliedFromHistory(t *testing.T) { } } -// 1945/AC18 (unit): a module that still runs from the kept declaration carries the kept declaration's filled paths of -// its files, so the filling writes them ("not named as filled" for claude-code and screen-lock, 2026-10-25). -func TestAModuleRunningFromTheKeptDeclarationKeepsItsFilledPaths(t *testing.T) { - b := newBed(t) - b.kept = map[string][]json.RawMessage{"claude-code": {file("claude-code", "settings", "")}} - b.env.KeptFilled = func() map[string][]string { - return map[string][]string{"claude-code": {"/tmp/r6/claude-code/settings"}} +// 1945/AC18 and issue 924 (unit): what runs of a module the agent has not taken yet carries the filled paths the +// filling judges by. From the kept declaration while the module has no have naming a composition ("not named as filled" +// for claude-code and screen-lock, 2026-10-25); from the composition its have names once it has one, the kept +// declaration being older, read at the start before any module settles and told to the filling then, so a file the +// newer composition fills is accepted on the first fill (power's three drop-ins on ace, g14 and shanks, 2026-10-29). +func TestWhatRunsBeforeTheAgentTakesAModuleNamesItsFilledPaths(t *testing.T) { + const old, added = "/tmp/r6/power/settings", "/tmp/r6/power/bound" + filledFile := func(id string) json.RawMessage { + return json.RawMessage(`{"id":"power.` + id + `","type":"file","module":"power","path":"/tmp/r6/power/` + id + + `","mode":"0644","owner-at-runtime":"owned"}`) } - a := New(b.env) - got := a.Everything() - if len(got) != 1 || got[0].Module != "claude-code" || strings.Join(got[0].Filled, ",") != "/tmp/r6/claude-code/settings" { - t.Fatalf("what runs from the kept declaration: %+v", got) + fp := fpOf("a") + ck, _ := delivery.ComposedKey(node, "power", fp) + applied := delivery.Composed{Node: node, Module: "power", FP: fp, Filled: []string{old, added}, + Resources: []json.RawMessage{filledFile("settings"), filledFile("bound")}} + _, appliedRaw, appliedDig := signedBy(t, applied, ck, controllerKey) + newer := applied + newer.Resources = append([]json.RawMessage{}, applied.Resources...) + newer.Resources = append(newer.Resources, file("power", "later", "1")) + _, newerRaw, _ := signedBy(t, newer, ck, controllerKey) + for _, tc := range []struct { + name string + // have is the module's have; bucket the value its composition key holds now, history the revisions kept. + have *Have + bucket []byte + history []Update + filled, told string + }{ + {name: "no have: the kept declaration's", filled: old}, + {name: "a have naming the bucket's value", have: &Have{Module: "power", FP: fp, Composed: appliedDig}, + bucket: appliedRaw, filled: old + "," + added, told: "the haves were read at the start"}, + {name: "a have naming a revision replaced since", have: &Have{Module: "power", FP: fp, Composed: appliedDig}, + bucket: newerRaw, history: []Update{{Key: ck, Value: appliedRaw, Revision: 3}, {Key: ck, Value: newerRaw, Revision: 7}}, + filled: old + "," + added, told: "the haves were read at the start"}, + {name: "a have naming a composition the bucket no longer keeps", have: &Have{Module: "power", FP: fp, Composed: appliedDig}, + bucket: newerRaw, filled: old, told: "the haves were read at the start"}, + } { + t.Run(tc.name, func(t *testing.T) { + b := newBed(t) + b.kept = map[string][]json.RawMessage{"power": {filledFile("settings")}} + b.env.KeptFilled = func() map[string][]string { return map[string][]string{"power": {old}} } + if tc.have != nil { + b.haves.stored = map[string]Have{"power": *tc.have} + } + if tc.bucket != nil { + b.comps.before(ck, tc.bucket) + } + b.env.Revisions = func(_ context.Context, key string) ([]Update, error) { return tc.history, nil } + var told []string + var mu sync.Mutex + b.env.RunsChanged = func(why string) { mu.Lock(); told = append(told, why); mu.Unlock() } + filledNow := func() string { + for _, c := range b.ag.Everything() { + if c.Module == "power" { + return strings.Join(c.Filled, ",") + } + } + return "" + } + if tc.told == "" { // no want, no have: nothing to start, and the kept declaration is what runs + a := New(b.env) + b.ag = a + if got := filledNow(); got != tc.filled { + t.Fatalf("filled %q, want %q", got, tc.filled) + } + return + } + stop := b.run() + defer stop() + eventually(t, "the filling was not told what runs at the start", func() bool { + mu.Lock() + defer mu.Unlock() + return len(told) > 0 && told[0] == tc.told + }) + if got := filledNow(); got != tc.filled { + t.Fatalf("filled %q, want %q", got, tc.filled) + } + if got := b.appliedList(); len(got) != 0 { + t.Fatalf("applied %v: nothing has a want", got) + } + }) } }