diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 6cf6335..d22910f 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -659,6 +659,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus machinesNow.Store(composer) // the machines a failing network part points at, by the assignments (issue 774) composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone env.Composed, env.SelfComposed, env.Policy, env.ComposedToday = composer, true, composer.Policy, composer.ComposedToday + env.WhyUnmade, env.Waiting = composer.WhyUnmade, composer.Waiting env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does session.NoteComposes(linked, nodeengineclient.ComposesOwn, "it composes its own modules, and applies no composition made elsewhere") ag = agent.New(env) diff --git a/internal/agent/agent.go b/internal/agent/agent.go index d92bed2..01cd6db 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -125,6 +125,11 @@ type Env struct { Wants Watcher Composed Watcher Haves Haves + // WhyUnmade is why the node's composer composed nothing of a module at its want's fingerprint, from its last pass; empty + // when it says nothing. Nil says nothing. Waiting tells the composer the agent waits for the composition of a module at + // a fingerprint it holds none of, so a composer that believes it handed it hands it again. Nil tells nobody. + WhyUnmade func(module string) string + Waiting func(module, fp string) // Apply applies one module's composition (scope NodeModule for the node's own), given the compositions the node's // other modules run so a path, unit or group they hold is not removed or taken. It is called with one module at a // time. @@ -1319,6 +1324,26 @@ func (a *Agent) HoldsNode() bool { return ok && h.Phase != nodeengineclient.PhaseRemoved } +// waitsForComposer says once per want and reason that the agent waits for the node's own composition of module at fp, and +// why the composer made none (Env.WhyUnmade), and tells the composer (Env.Waiting) so it hands again what it believes it +// handed. Called without a.mu held. +func (a *Agent) waitsForComposer(module, fp string) { + why := "the composer has made none yet" + if a.env.WhyUnmade != nil { + if w := a.env.WhyUnmade(module); w != "" { + why = w + } + } + line := fmt.Sprintf("%s waits for the composer: no composition of it at %s: %s", module, fp, why) + a.mu.Lock() + first := a.said["composer:"+module] != line + a.sayOnce("composer:"+module, line) + a.mu.Unlock() + if first && a.env.Waiting != nil { + a.env.Waiting(module, fp) + } +} + func (a *Agent) settleOne(ctx context.Context, module string) { a.mu.Lock() w := a.wants[module] @@ -1335,6 +1360,9 @@ func (a *Agent) settleOne(ctx context.Context, module string) { a.mu.Unlock() if a.env.SelfComposed { if !have || comp.Err != nil { + if !have { + a.waitsForComposer(module, w.Want.Fp) + } return // waits for the node's own composition of the want's build } // The composition the node composed for the want's build is the one it applies (ADR 0547 §2). diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 2b778d6..4285750 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -1935,3 +1935,23 @@ func testReconcileAppliesARunningModuleAgainAndLeavesTheHeldAndTheUnwantedAlone( t.Fatalf("a failed apply was tried again with no edge: %v", got) } } + +// Issue 1966 (novox, 2026-11-09): an agent that waits for its node's own composition of a want says so once, with why the +// composer made none, and tells the composer, which hands again what it believes it handed. +func TestAnAgentWaitingForTheComposerSaysSoOnceAndTellsIt(t *testing.T) { + b := newBed(t) + b.env.SelfComposed = true + var said []string + b.env.Say = func(s string) { said = append(said, s) } + b.env.WhyUnmade = func(string) string { return "no manifest in the composer's picture" } + var told []string + b.env.Waiting = func(module, fp string) { told = append(told, module+"@"+fp) } + a := New(b.env) + for range 2 { + a.waitsForComposer("web", fpOf("a")) + } + want := "web waits for the composer: no composition of it at " + fpOf("a") + ": no manifest in the composer's picture" + if len(said) != 1 || said[0] != want || len(told) != 1 || told[0] != "web@"+fpOf("a") { + t.Errorf("said %q, told %q; want %q once and web told once", said, told, want) + } +} diff --git a/internal/agent/composition.go b/internal/agent/composition.go index d626fb9..37c89d9 100644 --- a/internal/agent/composition.go +++ b/internal/agent/composition.go @@ -209,6 +209,11 @@ func (a *Agent) Composition(ctx context.Context, module, digest string, diff boo case digest != "" && read == nil: return view, fmt.Errorf("no revision of %s the bucket keeps has the digest %s; revisions lists those it keeps", key, digest) case read == nil && !haveLatest: + if a.env.SelfComposed && a.env.WhyUnmade != nil && module != NodeModule { + if why := a.env.WhyUnmade(module); why != "" { + return view, fmt.Errorf("the bucket holds no composition of %s at %s; the agent waits for the composer: %s", module, fp, why) + } + } return view, fmt.Errorf("the bucket holds no composition of %s at %s", module, fp) case read == nil: read = &latest diff --git a/internal/compose/frombus/frombus.go b/internal/compose/frombus/frombus.go index 58bda5b..069e370 100644 --- a/internal/compose/frombus/frombus.go +++ b/internal/compose/frombus/frombus.go @@ -167,6 +167,10 @@ type Scope struct { // Kept is told, in words, each module composed as what runs shows because its manifest does not yet say it (the tool // runner, the core's credentials; ADR 0601 §4), so the node says it. Nil: said nowhere. Kept func(module, words string) + // Unmade is told, in words, each module placed on the node that is composed from the indexer's manifest and not from a + // build, with why: the indexer's modules hold no build of the new form for it, or the builder holds none done at that + // fingerprint. Nil: said nowhere. + Unmade func(module, why string) // takeInHeld, set for the input a node answers a check with (ComposingInput), takes in the build of each module placed // on the node that its manifest's verdict or its decoder holds, into compose.Input.Holds, never into what the node composes: the // check's composer, judging the change's manifest itself, composes it at that build (issue 1921). @@ -930,6 +934,9 @@ func fromBuilds(ctx context.Context, r Reader, build func(ctx context.Context, m } for _, pm := range p.Modules { m, ok := modules[pm.Module] + if (!ok || m.Fp2 == "") && s.Unmade != nil && wanted[pm.Module] != "" { + s.Unmade(pm.Module, "the indexer's modules hold no build of the new form for it (fingerprint "+m.Fp2+"), so it is composed from the manifest") + } if !ok || m.Fp2 == "" || keptAt[pm.Module] != "" || compose.Folds(slices.Contains(s.InOwn, pm.Module), wanted[pm.Module]) { // A module it runs inside its own composition is composed there as before while it is folded: no build of the // new form is what it runs (Scope.InOwn). Wanted at a build of the new form it is no longer folded (compose.Folds, @@ -975,6 +982,13 @@ func fromBuilds(ctx context.Context, r Reader, build func(ctx context.Context, m stub(pm.Module, fmt.Errorf("frombus: %s is wanted at %s, of which the builder holds no done build yet; it "+ "keeps running what it runs", pm.Module, m.Fp2)) } + if s.Unmade != nil && wanted[pm.Module] != "" { + state := "none" + if b != nil { + state = string(b.State) + } + s.Unmade(pm.Module, "the builder holds no done build of it at "+m.Fp2+" (build: "+state+"), so it is composed from the indexer's manifest, which names no artifacts") + } continue // nothing of it runs: composed from the indexer's manifest, as before } resolved, err := built(pm.Module, m.Fp2, b) diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 989c5ac..d12b72a 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -154,6 +154,12 @@ type Composer struct { // the data items after it (publishData) do not read each placed module's build again over the bus (ADR 0577). builds map[string]builds.BuilderBuild handed map[string]string // composed key to the digest handed + // notTaken is, by module, why a want of this node's was not taken (it could not be decoded, or names no fingerprint). + notTaken map[string]string + // modulesRev is the revision of the last update of the indexer's modules taken: where the picture of them stands. + modulesRev uint64 + // rehand asks for a pass that hands again what the agent waits for (Waiting). + rehand chan struct{} // sealedOwn is each own secret's sealed value as first handed, by module/name and generation: a value sealed again // is other bytes, so the one first had is kept while the generation stands, and the composition stays the same. sealedOwn map[string]ownSealed @@ -238,6 +244,11 @@ type Pass struct { // holds is what holds each module Held names by its manifest's verdict or its decoder, with the build it would be // composed at (frombus.Scope.Holds), for LastHeld's words. holds map[string]compose.Hold + // Unmade is each module wanted that the pass composed no composition of at its want's fp, with why: the want not + // taken, the module not assigned here, no manifest in the composer's picture of the indexer's modules (and where that + // picture stands), or what the composition left it out for. A module held alone is in Held, not here. Said once per + // change as a journal line, so a want that waits for the composer says what for. + Unmade map[string]string `json:"unmade,omitempty"` // Handed is how many compositions the pass handed the agent that differ from what it handed before. Handed int `json:"handed"` // Again is when the node composes again on its own, after a seat or a read that did not answer; zero when it does @@ -340,7 +351,7 @@ func New(env Env) *Composer { } return &Composer{env: env, wants: map[string]meshwants.WantsByWantNodeModuleValue{}, assignments: map[string]meshmoduleassigner.Assignments{}, assignmentKeys: meshmoduleassignerclient.NewKeys(), records: map[string]nodeengine.NodeRecord{}, inOwn: map[string]bool{}, modules: map[string]meshcatalog.ModulesByModuleValue{}, values: map[string]map[string]any{}, - manifests: map[string][]byte{}, archives: map[string][]byte{}, builds: map[string]builds.BuilderBuild{}, handed: map[string]string{}, sealedOwn: map[string]ownSealed{}, + manifests: map[string][]byte{}, archives: map[string][]byte{}, builds: map[string]builds.BuilderBuild{}, handed: map[string]string{}, notTaken: map[string]string{}, rehand: make(chan struct{}, 1), sealedOwn: map[string]ownSealed{}, pkgs: map[string]compositions.Composed{}, asked: make(chan chan Pass, 8)} } @@ -544,6 +555,8 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er again = nil dirty = true c.env.Say(c.againLine()) + case <-c.rehand: + dirty = true case reply := <-c.asked: // Asked by hand (the verb compose): composed now; the growing bound is not restarted, so a pass that // fails again waits longer, not less. @@ -699,6 +712,13 @@ func (c *Composer) entry(bucket, key string) (v any, has, known bool) { } func (c *Composer) takeWant(u agent.Update) { + if line := c.takeWantLocked(u); line != "" { + c.sayOnceEach(line) + } +} + +// takeWantLocked takes the want in and answers a line to say when it is a want of this node's that is not taken. +func (c *Composer) takeWantLocked(u agent.Update) string { c.mu.Lock() defer c.mu.Unlock() if _, ok := meshwants.ParseWantsByWantNodeNodeKey(u.Key); ok { @@ -707,18 +727,30 @@ func (c *Composer) takeWant(u agent.Update) { } else { c.wants[agent.NodeModule] = meshwants.WantsByWantNodeModuleValue{} } - return + return "" } node, module, ok := meshwants.ParseWantsByWantNodeModuleKey(u.Key) if !ok || node != c.env.Node { - return + return "" } + delete(c.notTaken, module) w, err := meshwants.DecodeWantsByWantNodeModuleValue(u.Value) if u.Deleted || err != nil || w.Removed || w.Fp == "" { delete(c.wants, module) - return + switch { + case u.Deleted || (err == nil && w.Removed): + case err != nil: + c.notTaken[module] = "its want cannot be decoded: " + err.Error() + default: + c.notTaken[module] = "its want names no fingerprint" + } + if why := c.notTaken[module]; why != "" { + return "the node composes nothing of " + module + ": " + why + } + return "" } c.wants[module] = w + return "" } func (c *Composer) takeAssignments(u agent.Update) { @@ -844,6 +876,7 @@ func (c *Composer) takeModule(u agent.Update) { return } c.modules[name] = m + c.modulesRev = u.Revision } func (c *Composer) takeValue(u agent.Update) { @@ -946,6 +979,7 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { keptRuns := map[string][]string{} var keptMu sync.Mutex var holds map[string]compose.Hold + fromManifest := map[string]string{} // by module, why it is composed from the indexer's manifest and not a build scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running, Kept: func(module, words string) { keptMu.Lock() defer keptMu.Unlock() @@ -953,12 +987,16 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { keptRuns[module] = append(keptRuns[module], words) } c.sayOnceEach(words) - }, Holds: func(h map[string]compose.Hold) { holds = h }} + }, Holds: func(h map[string]compose.Hold) { holds = h }, Unmade: func(module, why string) { + keptMu.Lock() + defer keptMu.Unlock() + fromManifest[module] = why + }} c.mu.Lock() c.dataScope = frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn} // the data items are worked out as composed c.mu.Unlock() missed := c.missed.Load() - compositions, _, held, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps) + compositions, skipped, held, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps) err = c.missedIn(missed, err) // A bus that does not answer for a moment (a reload of its accounts pauses it a few seconds) is no refusal: the pass is // tried again after a growing wait before anything is marked (issue 1126). @@ -970,7 +1008,7 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { } if ctx.Err() == nil { missed = c.missed.Load() - compositions, _, held, err = frombus.Compose(ctx, r, c.env.Mesh, scope, fps) + compositions, skipped, held, err = frombus.Compose(ctx, r, c.env.Mesh, scope, fps) err = c.missedIn(missed, err) } } @@ -1013,6 +1051,8 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { } c.paid() pass.Held, pass.holds = held, holds + pass.Unmade = c.unmadeOf(fps, compositions, held, skipped, own, inOwn, fromManifest, &keptMu) + c.sayUnmade(pass.Unmade) // The provisioner client's account the node runs with, dropped by this pass, is said: every filled write is refused // under a declaration that names none (filling.JudgeAccount), and no module is handed to it (issue 774, ace, // 2026-11-04 01:29). A placed client keeps its account while it is held (compose.Node.RunningProvisioner). @@ -2028,3 +2068,98 @@ func (c *Composer) seatsOf(ctx context.Context, r reader, module string) []strin } return seats } + +// unmadeOf is each module wanted at a fingerprint (fps) of which the pass made no composition at it, with why, apart from +// the modules held alone (Pass.Unmade), and each want of this node's that was not taken. +func (c *Composer) unmadeOf(fps map[string]string, comps map[string]engcomp.Composed, held map[string]string, skipped []string, + own meshmoduleassigner.Assignments, inOwn []string, fromManifest map[string]string, mu *sync.Mutex) map[string]string { + made := map[string]string{} + for _, comp := range comps { + if comp.Module != "" { + made[comp.Module] = comp.FP + } + } + placed := map[string]bool{} + for _, m := range own.Modules { + placed[m.Module] = true + } + mu.Lock() + defer mu.Unlock() + c.mu.Lock() + defer c.mu.Unlock() + out := map[string]string{} + for module, fp := range fps { + if _, isHeld := held[module]; isHeld || made[module] == fp { + continue + } + var why string + switch m, in := c.modules[module]; { + case compose.Folds(slices.Contains(inOwn, module), fp): + why = "it is composed inside the node's own composition at " + fp + case !placed[module]: + why = fmt.Sprintf("the seat's assignment places no %s on this node (%d modules placed)", module, len(own.Modules)) + case !in: + why = fmt.Sprintf("no manifest in the composer's picture of the indexer's modules (%d modules, last update at revision %d)", + len(c.modules), c.modulesRev) + case slices.Contains(skipped, module): + why = "the composition skipped it: the declaration gives it no fingerprint (indexer's picture: " + m.Fp2 + ")" + default: + why = fmt.Sprintf("the composition made nothing of it at %s (indexer's picture: %s, revision %d)", fp, m.Fp2, c.modulesRev) + } + if w, ok := fromManifest[module]; ok { + why += "; " + w + } + out[module] = why + } + for module, why := range c.notTaken { + if _, wanted := fps[module]; !wanted { + out[module] = "want not taken: " + why + } + } + if len(out) == 0 { + return nil + } + return out +} + +// sayUnmade says each change of what the pass composed none of, once. +func (c *Composer) sayUnmade(unmade map[string]string) { + if len(unmade) == 0 { + return + } + var lines []string + for _, m := range heldModules(unmade) { + lines = append(lines, m+": "+unmade[m]) + } + c.sayOnceEach("the node composes nothing of " + strings.Join(lines, "; ")) +} + +// Waiting is the agent's word that it waits for the composition of module at fp, which it holds none of. When the composer +// believes it handed that composition, it forgets so and composes again, so the agent is handed it again: what was handed +// is no more trusted over what the agent holds (issue 1966, novox: handed zero, the agent waiting). +func (c *Composer) Waiting(module, fp string) { + key, err := engcomp.Key(c.env.Node, module, fp) + if err != nil { + return + } + c.mu.Lock() + _, believed := c.handed[key] + delete(c.handed, key) + c.mu.Unlock() + if !believed { + return + } + c.env.Say(fmt.Sprintf("the agent waits for the composition of %s at %s, which the node believed it handed: handed again", module, fp)) + select { + case c.rehand <- struct{}{}: + default: + } +} + +// WhyUnmade is why the last pass composed nothing of module at its want's fingerprint (Pass.Unmade); empty when it said +// none, or no pass was made. +func (c *Composer) WhyUnmade(module string) string { + c.mu.Lock() + defer c.mu.Unlock() + return c.last.Unmade[module] +} diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index 701aa48..b7617b7 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -2090,3 +2090,59 @@ func (b *patternBus) putAssigned(vals map[string][]byte, rev uint64) { b.put(buckets.Assignments, k, vals[k], rev) } } + +// Issue 1966 (novox, 2026-11-09): a want the composer made nothing of was said nowhere, and a composition it believed +// handed that the agent never held stayed unhanded. Every pass names each wanted module it composed nothing of, with why, +// in Pass.Unmade and as a journal line; a want not taken is said; and the agent's word that it waits hands the +// composition again. +func TestAWantTheNodeMakesNothingOfSaysWhyAndAWaitingAgentIsHandedItAgain(t *testing.T) { + b := newFakeBus() + fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) + gfp := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64) + digest := meshcatalogclient.ManifestDigest([]byte(lemurs)) + man, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "lemurs", Manifest: lemurs}) + key, _ := meshcatalogclient.ManifestsKey(digest) + b.kv[buckets.Manifests+"/"+key] = man + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := New(b.env()) + out, err := c.Watch(ctx, "n1.>") + if err != nil { + t.Fatal(err) + } + place := assignedValues(t, meshmoduleassigner.Assignments{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}, + Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}}}) + have, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}}) + mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}) + want, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Self: true}) + ghost, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: gfp, Self: true}) + nofp, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Self: true}) + b.putAssigned(t, place) + b.put(t, buckets.Have, "n1.node", have) + b.put(t, buckets.Modules, "lemurs", mod) + b.put(t, buckets.Wants, "want.n1.lemurs", want) + b.put(t, buckets.Wants, "want.n1.ghost", ghost) + b.put(t, buckets.Wants, "want.n1.nofp", nofp) + for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, buckets.Modules, ResolvedBucket, builds.BuilderBuildsBucket} { + b.send(bucket, agent.Update{Caught: true}) + } + for !next(t, out).Caught { + } + why := map[string]string{"ghost": c.WhyUnmade("ghost"), "nofp": c.WhyUnmade("nofp"), "lemurs": c.WhyUnmade("lemurs")} + if !strings.Contains(why["ghost"], "places no ghost") || !strings.Contains(why["nofp"], "names no fingerprint") || why["lemurs"] != "" { + t.Errorf("the pass says it composed nothing of %q", why) + } + b.mu.Lock() + said := strings.Join(b.said, "\n") + b.mu.Unlock() + if !strings.Contains(said, "the node composes nothing of ghost: ") { + t.Errorf("the journal does not say why ghost is not composed: %q", said) + } + // The agent waits for what the node believes it handed: handed again, and a wait for what was never handed does nothing. + c.Waiting("ghost", gfp) + quiet(t, out) + c.Waiting("lemurs", fp) + if u := next(t, out); !strings.Contains(u.Key, "lemurs") { + t.Errorf("handed %s, want lemurs's composition again", u.Key) + } +}