From 5f37a43b91f6c39507b46e408d2f5ca7218fad4f Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 08:58:33 +0100 Subject: [PATCH] The node says why it composes nothing of a want, and hands again what its agent waits for (issue 1966) Every pass names each wanted module it made no composition of, with why (Pass.Unmade, the verb compose, one journal line per change); a want not taken is said; the agent says once per want that it waits for the composer and why, and tells it, so a composition the composer believed handed is handed again. Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- cmd/mesh-host/nodeagent.go | 1 + internal/agent/agent.go | 28 +++++ internal/agent/agent_test.go | 20 +++ internal/agent/composition.go | 5 + internal/compose/frombus/frombus.go | 14 +++ internal/selfcompose/selfcompose.go | 149 +++++++++++++++++++++-- internal/selfcompose/selfcompose_test.go | 56 +++++++++ 7 files changed, 266 insertions(+), 7 deletions(-) diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 6cf6335..d22910f 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -659,6 +659,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus machinesNow.Store(composer) // the machines a failing network part points at, by the assignments (issue 774) composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone env.Composed, env.SelfComposed, env.Policy, env.ComposedToday = composer, true, composer.Policy, composer.ComposedToday + env.WhyUnmade, env.Waiting = composer.WhyUnmade, composer.Waiting env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does session.NoteComposes(linked, nodeengineclient.ComposesOwn, "it composes its own modules, and applies no composition made elsewhere") ag = agent.New(env) diff --git a/internal/agent/agent.go b/internal/agent/agent.go index d92bed2..01cd6db 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -125,6 +125,11 @@ type Env struct { Wants Watcher Composed Watcher Haves Haves + // WhyUnmade is why the node's composer composed nothing of a module at its want's fingerprint, from its last pass; empty + // when it says nothing. Nil says nothing. Waiting tells the composer the agent waits for the composition of a module at + // a fingerprint it holds none of, so a composer that believes it handed it hands it again. Nil tells nobody. + WhyUnmade func(module string) string + Waiting func(module, fp string) // Apply applies one module's composition (scope NodeModule for the node's own), given the compositions the node's // other modules run so a path, unit or group they hold is not removed or taken. It is called with one module at a // time. @@ -1319,6 +1324,26 @@ func (a *Agent) HoldsNode() bool { return ok && h.Phase != nodeengineclient.PhaseRemoved } +// waitsForComposer says once per want and reason that the agent waits for the node's own composition of module at fp, and +// why the composer made none (Env.WhyUnmade), and tells the composer (Env.Waiting) so it hands again what it believes it +// handed. Called without a.mu held. +func (a *Agent) waitsForComposer(module, fp string) { + why := "the composer has made none yet" + if a.env.WhyUnmade != nil { + if w := a.env.WhyUnmade(module); w != "" { + why = w + } + } + line := fmt.Sprintf("%s waits for the composer: no composition of it at %s: %s", module, fp, why) + a.mu.Lock() + first := a.said["composer:"+module] != line + a.sayOnce("composer:"+module, line) + a.mu.Unlock() + if first && a.env.Waiting != nil { + a.env.Waiting(module, fp) + } +} + func (a *Agent) settleOne(ctx context.Context, module string) { a.mu.Lock() w := a.wants[module] @@ -1335,6 +1360,9 @@ func (a *Agent) settleOne(ctx context.Context, module string) { a.mu.Unlock() if a.env.SelfComposed { if !have || comp.Err != nil { + if !have { + a.waitsForComposer(module, w.Want.Fp) + } return // waits for the node's own composition of the want's build } // The composition the node composed for the want's build is the one it applies (ADR 0547 §2). diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 2b778d6..4285750 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -1935,3 +1935,23 @@ func testReconcileAppliesARunningModuleAgainAndLeavesTheHeldAndTheUnwantedAlone( t.Fatalf("a failed apply was tried again with no edge: %v", got) } } + +// Issue 1966 (novox, 2026-11-09): an agent that waits for its node's own composition of a want says so once, with why the +// composer made none, and tells the composer, which hands again what it believes it handed. +func TestAnAgentWaitingForTheComposerSaysSoOnceAndTellsIt(t *testing.T) { + b := newBed(t) + b.env.SelfComposed = true + var said []string + b.env.Say = func(s string) { said = append(said, s) } + b.env.WhyUnmade = func(string) string { return "no manifest in the composer's picture" } + var told []string + b.env.Waiting = func(module, fp string) { told = append(told, module+"@"+fp) } + a := New(b.env) + for range 2 { + a.waitsForComposer("web", fpOf("a")) + } + want := "web waits for the composer: no composition of it at " + fpOf("a") + ": no manifest in the composer's picture" + if len(said) != 1 || said[0] != want || len(told) != 1 || told[0] != "web@"+fpOf("a") { + t.Errorf("said %q, told %q; want %q once and web told once", said, told, want) + } +} diff --git a/internal/agent/composition.go b/internal/agent/composition.go index d626fb9..37c89d9 100644 --- a/internal/agent/composition.go +++ b/internal/agent/composition.go @@ -209,6 +209,11 @@ func (a *Agent) Composition(ctx context.Context, module, digest string, diff boo case digest != "" && read == nil: return view, fmt.Errorf("no revision of %s the bucket keeps has the digest %s; revisions lists those it keeps", key, digest) case read == nil && !haveLatest: + if a.env.SelfComposed && a.env.WhyUnmade != nil && module != NodeModule { + if why := a.env.WhyUnmade(module); why != "" { + return view, fmt.Errorf("the bucket holds no composition of %s at %s; the agent waits for the composer: %s", module, fp, why) + } + } return view, fmt.Errorf("the bucket holds no composition of %s at %s", module, fp) case read == nil: read = &latest diff --git a/internal/compose/frombus/frombus.go b/internal/compose/frombus/frombus.go index 58bda5b..069e370 100644 --- a/internal/compose/frombus/frombus.go +++ b/internal/compose/frombus/frombus.go @@ -167,6 +167,10 @@ type Scope struct { // Kept is told, in words, each module composed as what runs shows because its manifest does not yet say it (the tool // runner, the core's credentials; ADR 0601 §4), so the node says it. Nil: said nowhere. Kept func(module, words string) + // Unmade is told, in words, each module placed on the node that is composed from the indexer's manifest and not from a + // build, with why: the indexer's modules hold no build of the new form for it, or the builder holds none done at that + // fingerprint. Nil: said nowhere. + Unmade func(module, why string) // takeInHeld, set for the input a node answers a check with (ComposingInput), takes in the build of each module placed // on the node that its manifest's verdict or its decoder holds, into compose.Input.Holds, never into what the node composes: the // check's composer, judging the change's manifest itself, composes it at that build (issue 1921). @@ -930,6 +934,9 @@ func fromBuilds(ctx context.Context, r Reader, build func(ctx context.Context, m } for _, pm := range p.Modules { m, ok := modules[pm.Module] + if (!ok || m.Fp2 == "") && s.Unmade != nil && wanted[pm.Module] != "" { + s.Unmade(pm.Module, "the indexer's modules hold no build of the new form for it (fingerprint "+m.Fp2+"), so it is composed from the manifest") + } if !ok || m.Fp2 == "" || keptAt[pm.Module] != "" || compose.Folds(slices.Contains(s.InOwn, pm.Module), wanted[pm.Module]) { // A module it runs inside its own composition is composed there as before while it is folded: no build of the // new form is what it runs (Scope.InOwn). Wanted at a build of the new form it is no longer folded (compose.Folds, @@ -975,6 +982,13 @@ func fromBuilds(ctx context.Context, r Reader, build func(ctx context.Context, m stub(pm.Module, fmt.Errorf("frombus: %s is wanted at %s, of which the builder holds no done build yet; it "+ "keeps running what it runs", pm.Module, m.Fp2)) } + if s.Unmade != nil && wanted[pm.Module] != "" { + state := "none" + if b != nil { + state = string(b.State) + } + s.Unmade(pm.Module, "the builder holds no done build of it at "+m.Fp2+" (build: "+state+"), so it is composed from the indexer's manifest, which names no artifacts") + } continue // nothing of it runs: composed from the indexer's manifest, as before } resolved, err := built(pm.Module, m.Fp2, b) diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 989c5ac..d12b72a 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -154,6 +154,12 @@ type Composer struct { // the data items after it (publishData) do not read each placed module's build again over the bus (ADR 0577). builds map[string]builds.BuilderBuild handed map[string]string // composed key to the digest handed + // notTaken is, by module, why a want of this node's was not taken (it could not be decoded, or names no fingerprint). + notTaken map[string]string + // modulesRev is the revision of the last update of the indexer's modules taken: where the picture of them stands. + modulesRev uint64 + // rehand asks for a pass that hands again what the agent waits for (Waiting). + rehand chan struct{} // sealedOwn is each own secret's sealed value as first handed, by module/name and generation: a value sealed again // is other bytes, so the one first had is kept while the generation stands, and the composition stays the same. sealedOwn map[string]ownSealed @@ -238,6 +244,11 @@ type Pass struct { // holds is what holds each module Held names by its manifest's verdict or its decoder, with the build it would be // composed at (frombus.Scope.Holds), for LastHeld's words. holds map[string]compose.Hold + // Unmade is each module wanted that the pass composed no composition of at its want's fp, with why: the want not + // taken, the module not assigned here, no manifest in the composer's picture of the indexer's modules (and where that + // picture stands), or what the composition left it out for. A module held alone is in Held, not here. Said once per + // change as a journal line, so a want that waits for the composer says what for. + Unmade map[string]string `json:"unmade,omitempty"` // Handed is how many compositions the pass handed the agent that differ from what it handed before. Handed int `json:"handed"` // Again is when the node composes again on its own, after a seat or a read that did not answer; zero when it does @@ -340,7 +351,7 @@ func New(env Env) *Composer { } return &Composer{env: env, wants: map[string]meshwants.WantsByWantNodeModuleValue{}, assignments: map[string]meshmoduleassigner.Assignments{}, assignmentKeys: meshmoduleassignerclient.NewKeys(), records: map[string]nodeengine.NodeRecord{}, inOwn: map[string]bool{}, modules: map[string]meshcatalog.ModulesByModuleValue{}, values: map[string]map[string]any{}, - manifests: map[string][]byte{}, archives: map[string][]byte{}, builds: map[string]builds.BuilderBuild{}, handed: map[string]string{}, sealedOwn: map[string]ownSealed{}, + manifests: map[string][]byte{}, archives: map[string][]byte{}, builds: map[string]builds.BuilderBuild{}, handed: map[string]string{}, notTaken: map[string]string{}, rehand: make(chan struct{}, 1), sealedOwn: map[string]ownSealed{}, pkgs: map[string]compositions.Composed{}, asked: make(chan chan Pass, 8)} } @@ -544,6 +555,8 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er again = nil dirty = true c.env.Say(c.againLine()) + case <-c.rehand: + dirty = true case reply := <-c.asked: // Asked by hand (the verb compose): composed now; the growing bound is not restarted, so a pass that // fails again waits longer, not less. @@ -699,6 +712,13 @@ func (c *Composer) entry(bucket, key string) (v any, has, known bool) { } func (c *Composer) takeWant(u agent.Update) { + if line := c.takeWantLocked(u); line != "" { + c.sayOnceEach(line) + } +} + +// takeWantLocked takes the want in and answers a line to say when it is a want of this node's that is not taken. +func (c *Composer) takeWantLocked(u agent.Update) string { c.mu.Lock() defer c.mu.Unlock() if _, ok := meshwants.ParseWantsByWantNodeNodeKey(u.Key); ok { @@ -707,18 +727,30 @@ func (c *Composer) takeWant(u agent.Update) { } else { c.wants[agent.NodeModule] = meshwants.WantsByWantNodeModuleValue{} } - return + return "" } node, module, ok := meshwants.ParseWantsByWantNodeModuleKey(u.Key) if !ok || node != c.env.Node { - return + return "" } + delete(c.notTaken, module) w, err := meshwants.DecodeWantsByWantNodeModuleValue(u.Value) if u.Deleted || err != nil || w.Removed || w.Fp == "" { delete(c.wants, module) - return + switch { + case u.Deleted || (err == nil && w.Removed): + case err != nil: + c.notTaken[module] = "its want cannot be decoded: " + err.Error() + default: + c.notTaken[module] = "its want names no fingerprint" + } + if why := c.notTaken[module]; why != "" { + return "the node composes nothing of " + module + ": " + why + } + return "" } c.wants[module] = w + return "" } func (c *Composer) takeAssignments(u agent.Update) { @@ -844,6 +876,7 @@ func (c *Composer) takeModule(u agent.Update) { return } c.modules[name] = m + c.modulesRev = u.Revision } func (c *Composer) takeValue(u agent.Update) { @@ -946,6 +979,7 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { keptRuns := map[string][]string{} var keptMu sync.Mutex var holds map[string]compose.Hold + fromManifest := map[string]string{} // by module, why it is composed from the indexer's manifest and not a build scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running, Kept: func(module, words string) { keptMu.Lock() defer keptMu.Unlock() @@ -953,12 +987,16 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { keptRuns[module] = append(keptRuns[module], words) } c.sayOnceEach(words) - }, Holds: func(h map[string]compose.Hold) { holds = h }} + }, Holds: func(h map[string]compose.Hold) { holds = h }, Unmade: func(module, why string) { + keptMu.Lock() + defer keptMu.Unlock() + fromManifest[module] = why + }} c.mu.Lock() c.dataScope = frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn} // the data items are worked out as composed c.mu.Unlock() missed := c.missed.Load() - compositions, _, held, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps) + compositions, skipped, held, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps) err = c.missedIn(missed, err) // A bus that does not answer for a moment (a reload of its accounts pauses it a few seconds) is no refusal: the pass is // tried again after a growing wait before anything is marked (issue 1126). @@ -970,7 +1008,7 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { } if ctx.Err() == nil { missed = c.missed.Load() - compositions, _, held, err = frombus.Compose(ctx, r, c.env.Mesh, scope, fps) + compositions, skipped, held, err = frombus.Compose(ctx, r, c.env.Mesh, scope, fps) err = c.missedIn(missed, err) } } @@ -1013,6 +1051,8 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { } c.paid() pass.Held, pass.holds = held, holds + pass.Unmade = c.unmadeOf(fps, compositions, held, skipped, own, inOwn, fromManifest, &keptMu) + c.sayUnmade(pass.Unmade) // The provisioner client's account the node runs with, dropped by this pass, is said: every filled write is refused // under a declaration that names none (filling.JudgeAccount), and no module is handed to it (issue 774, ace, // 2026-11-04 01:29). A placed client keeps its account while it is held (compose.Node.RunningProvisioner). @@ -2028,3 +2068,98 @@ func (c *Composer) seatsOf(ctx context.Context, r reader, module string) []strin } return seats } + +// unmadeOf is each module wanted at a fingerprint (fps) of which the pass made no composition at it, with why, apart from +// the modules held alone (Pass.Unmade), and each want of this node's that was not taken. +func (c *Composer) unmadeOf(fps map[string]string, comps map[string]engcomp.Composed, held map[string]string, skipped []string, + own meshmoduleassigner.Assignments, inOwn []string, fromManifest map[string]string, mu *sync.Mutex) map[string]string { + made := map[string]string{} + for _, comp := range comps { + if comp.Module != "" { + made[comp.Module] = comp.FP + } + } + placed := map[string]bool{} + for _, m := range own.Modules { + placed[m.Module] = true + } + mu.Lock() + defer mu.Unlock() + c.mu.Lock() + defer c.mu.Unlock() + out := map[string]string{} + for module, fp := range fps { + if _, isHeld := held[module]; isHeld || made[module] == fp { + continue + } + var why string + switch m, in := c.modules[module]; { + case compose.Folds(slices.Contains(inOwn, module), fp): + why = "it is composed inside the node's own composition at " + fp + case !placed[module]: + why = fmt.Sprintf("the seat's assignment places no %s on this node (%d modules placed)", module, len(own.Modules)) + case !in: + why = fmt.Sprintf("no manifest in the composer's picture of the indexer's modules (%d modules, last update at revision %d)", + len(c.modules), c.modulesRev) + case slices.Contains(skipped, module): + why = "the composition skipped it: the declaration gives it no fingerprint (indexer's picture: " + m.Fp2 + ")" + default: + why = fmt.Sprintf("the composition made nothing of it at %s (indexer's picture: %s, revision %d)", fp, m.Fp2, c.modulesRev) + } + if w, ok := fromManifest[module]; ok { + why += "; " + w + } + out[module] = why + } + for module, why := range c.notTaken { + if _, wanted := fps[module]; !wanted { + out[module] = "want not taken: " + why + } + } + if len(out) == 0 { + return nil + } + return out +} + +// sayUnmade says each change of what the pass composed none of, once. +func (c *Composer) sayUnmade(unmade map[string]string) { + if len(unmade) == 0 { + return + } + var lines []string + for _, m := range heldModules(unmade) { + lines = append(lines, m+": "+unmade[m]) + } + c.sayOnceEach("the node composes nothing of " + strings.Join(lines, "; ")) +} + +// Waiting is the agent's word that it waits for the composition of module at fp, which it holds none of. When the composer +// believes it handed that composition, it forgets so and composes again, so the agent is handed it again: what was handed +// is no more trusted over what the agent holds (issue 1966, novox: handed zero, the agent waiting). +func (c *Composer) Waiting(module, fp string) { + key, err := engcomp.Key(c.env.Node, module, fp) + if err != nil { + return + } + c.mu.Lock() + _, believed := c.handed[key] + delete(c.handed, key) + c.mu.Unlock() + if !believed { + return + } + c.env.Say(fmt.Sprintf("the agent waits for the composition of %s at %s, which the node believed it handed: handed again", module, fp)) + select { + case c.rehand <- struct{}{}: + default: + } +} + +// WhyUnmade is why the last pass composed nothing of module at its want's fingerprint (Pass.Unmade); empty when it said +// none, or no pass was made. +func (c *Composer) WhyUnmade(module string) string { + c.mu.Lock() + defer c.mu.Unlock() + return c.last.Unmade[module] +} diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index 701aa48..b7617b7 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -2090,3 +2090,59 @@ func (b *patternBus) putAssigned(vals map[string][]byte, rev uint64) { b.put(buckets.Assignments, k, vals[k], rev) } } + +// Issue 1966 (novox, 2026-11-09): a want the composer made nothing of was said nowhere, and a composition it believed +// handed that the agent never held stayed unhanded. Every pass names each wanted module it composed nothing of, with why, +// in Pass.Unmade and as a journal line; a want not taken is said; and the agent's word that it waits hands the +// composition again. +func TestAWantTheNodeMakesNothingOfSaysWhyAndAWaitingAgentIsHandedItAgain(t *testing.T) { + b := newFakeBus() + fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) + gfp := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64) + digest := meshcatalogclient.ManifestDigest([]byte(lemurs)) + man, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "lemurs", Manifest: lemurs}) + key, _ := meshcatalogclient.ManifestsKey(digest) + b.kv[buckets.Manifests+"/"+key] = man + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := New(b.env()) + out, err := c.Watch(ctx, "n1.>") + if err != nil { + t.Fatal(err) + } + place := assignedValues(t, meshmoduleassigner.Assignments{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}, + Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}}}) + have, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}}) + mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}) + want, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Self: true}) + ghost, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: gfp, Self: true}) + nofp, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Self: true}) + b.putAssigned(t, place) + b.put(t, buckets.Have, "n1.node", have) + b.put(t, buckets.Modules, "lemurs", mod) + b.put(t, buckets.Wants, "want.n1.lemurs", want) + b.put(t, buckets.Wants, "want.n1.ghost", ghost) + b.put(t, buckets.Wants, "want.n1.nofp", nofp) + for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, buckets.Modules, ResolvedBucket, builds.BuilderBuildsBucket} { + b.send(bucket, agent.Update{Caught: true}) + } + for !next(t, out).Caught { + } + why := map[string]string{"ghost": c.WhyUnmade("ghost"), "nofp": c.WhyUnmade("nofp"), "lemurs": c.WhyUnmade("lemurs")} + if !strings.Contains(why["ghost"], "places no ghost") || !strings.Contains(why["nofp"], "names no fingerprint") || why["lemurs"] != "" { + t.Errorf("the pass says it composed nothing of %q", why) + } + b.mu.Lock() + said := strings.Join(b.said, "\n") + b.mu.Unlock() + if !strings.Contains(said, "the node composes nothing of ghost: ") { + t.Errorf("the journal does not say why ghost is not composed: %q", said) + } + // The agent waits for what the node believes it handed: handed again, and a wait for what was never handed does nothing. + c.Waiting("ghost", gfp) + quiet(t, out) + c.Waiting("lemurs", fp) + if u := next(t, out); !strings.Contains(u.Key, "lemurs") { + t.Errorf("handed %s, want lemurs's composition again", u.Key) + } +}