From 5cc26e2659e5c79ef2ef4cc4d690166bbf7d04bc Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 18 Nov 2026 01:02:10 +0100 Subject: [PATCH] selfcompose: the one place says its source at every caller, reads the have's build, answers HoldsData; the tool composition reads it outside the agent's lock (A-484, A-482) Builds: design 47 G A-484, A-484a, A-482 --- internal/agent/agent_test.go | 1 + internal/agent/composition.go | 9 ++- internal/selfcompose/entries.go | 21 ++++-- internal/selfcompose/entries_test.go | 109 +++++++++++++++++++++++++++ internal/selfcompose/running.go | 18 ++--- internal/selfcompose/running_test.go | 83 -------------------- internal/selfcompose/selfcompose.go | 33 ++++---- 7 files changed, 152 insertions(+), 122 deletions(-) delete mode 100644 internal/selfcompose/running_test.go diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index f4f8e4c2..fdc8c93b 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -1720,6 +1720,7 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes( } // Where the one place says what runs of it (A-484), the tool answers it though the want's build has no composition. b.ag.env.RunningManifest = func(string) (string, string, string) { + _ = b.ag.Running() // the one place reads what the agent runs: the tool must not hold the agent's lock while it does return "the module's trunk manifest in the catalog", "src2:t", "the manifest of the build ghost runs here is read from the module's trunk manifest in the catalog" } if view, err := b.ag.Composition(context.Background(), "ghost", "", false); err != nil || view.RunsManifest == nil || diff --git a/internal/agent/composition.go b/internal/agent/composition.go index b570f5f7..9856382f 100644 --- a/internal/agent/composition.go +++ b/internal/agent/composition.go @@ -189,10 +189,6 @@ func (a *Agent) Composition(ctx context.Context, module, digest string, diff boo Held: att.Held, AnsweredRevision: att.AnsweredRev, Transient: att.Transient} } view.Started = a.checked[module] - if a.env.RunningManifest != nil && module != NodeModule { - src, rfp, words := a.env.RunningManifest(module) - view.RunsManifest = &RunsManifestView{Source: src, FP: rfp, Words: words} - } latest, haveLatest := a.compositionFor(module, fp) r, ran := a.running[module] from, known := a.runningComposition(module) @@ -205,6 +201,11 @@ func (a *Agent) Composition(ctx context.Context, module, digest string, diff boo } } a.mu.Unlock() + // Read outside the lock: the one place reads what the agent runs (Agent.Running takes it), and nothing it does is asked of the bus. + if a.env.RunningManifest != nil && module != NodeModule { + src, rfp, words := a.env.RunningManifest(module) + view.RunsManifest = &RunsManifestView{Source: src, FP: rfp, Words: words} + } if module != NodeModule && fp == "" { return view, fmt.Errorf("%s has no want and no have on %s, so no build of it is composed here", module, a.env.Node) } diff --git a/internal/selfcompose/entries.go b/internal/selfcompose/entries.go index 6bb471a9..c3b5da11 100644 --- a/internal/selfcompose/entries.go +++ b/internal/selfcompose/entries.go @@ -413,6 +413,9 @@ func (c *Composer) layModules() { // layAnswer lays a module from the manifest the one place read of the build it runs (A-484): the entry's own fields, or the // trunk's answer, which holds none of an entry's checks. layAnswer := func(module string, a runningAnswer) bool { + if a.Said != "" { + said = append(said, a.Said) + } switch { case !a.Known: return false @@ -479,17 +482,15 @@ func (c *Composer) layModules() { } // Running here with no entry of its wanted build and none of the newest: it stands, for resolution alone, as the manifest of // the build it runs, from the one place (A-484), so the module that requires what it provides is no "nothing provides"; - // its want stays held. A running build whose entry is held is laid where it is heard (above) and is read by a held module's offer - // through the one place alone, so it is not laid again here. + // its want stays held. An answer from the entry of the running build is not laid here: that entry is laid where its module is heard + // (above), and laying it for a module that is not (heard nowhere yet, its want moving) made the node compose and apply the move twice + // (TestNatsTheBusServerIsTakenOver..., one run in five); a held module's offer reads that entry through the one place alone. if _, laid := modules[module]; !laid && c.countsAsRunning(module) { if a := c.runningLocked(running, c.env.Node, module, ""); a.Source != SourceEntry && layAnswer(module, a) { m := modules[module] m.Fp2 = w.Fp modules[module] = m said = append(said, fmt.Sprintf("%s runs a build whose entry the node does not hold, so it counts the manifest read from %s; its want stays held", module, a.Source)) - if a.Said != "" { - said = append(said, a.Said) - } continue } } @@ -562,6 +563,7 @@ func (c *Composer) needsAddressOf(node, module string) bool { func (c *Composer) askTrunkBuilds() (wait func()) { type ask struct{ node, module, running string } var asks []ask + var sayLines []string var running []engcomp.Composed if c.env.Running != nil { running = c.env.Running() @@ -582,7 +584,11 @@ func (c *Composer) askTrunkBuilds() (wait func()) { } else if !c.needsAddressOf(node, module) || !c.lacksEntry(node, module) { continue } - if a := c.runningLocked(running, node, module, ""); a.NeedTrunk { + a := c.runningLocked(running, node, module, "") + if a.Said != "" { + sayLines = append(sayLines, a.Said) + } + if a.NeedTrunk { if had, asked := c.trunkBuilds[module]; !asked || had.running != a.RunFP { asks = append(asks, ask{node, module, a.RunFP}) if c.trunkAsking == nil { @@ -594,6 +600,9 @@ func (c *Composer) askTrunkBuilds() (wait func()) { } } c.mu.Unlock() + for _, line := range sayLines { + c.sayOnceEach(line) + } // Asked beside the pass, a few at a time: no pass waits on the catalog half's answer, and a module that waits for it is only // the one needing that address (A-461e). An answer asks a pass. var all sync.WaitGroup diff --git a/internal/selfcompose/entries_test.go b/internal/selfcompose/entries_test.go index 3fa58a39..937214b7 100644 --- a/internal/selfcompose/entries_test.go +++ b/internal/selfcompose/entries_test.go @@ -449,6 +449,115 @@ func TestRunningBuildEntryKeptAndHandedAgain(t *testing.T) { } } + // A-484, A-484a, A-482: the manifest of the build a module runs here is answered in one place, from the entry of the running build, + // else the manifest its running statement names, else the module's trunk manifest; the composition says whether it keeps data. It + // says its source once per module and source, whoever asks, and where none answers its words name the three. + { + running := meshcatalogclient.SourcePrefix2 + strings.Repeat("1", 64) + other := meshcatalogclient.SourcePrefix2 + strings.Repeat("2", 64) + body := `{"module":"xorg","version":"1","provides":[{"name":"x11-display"}]}` + for _, tc := range []struct { + name string + entry string // the entry held, by fingerprint ("running", "other") or none + composition string // the composition this node runs: its build ("running", "other") or none + data bool + trunk bool + statement bool // the running event names a manifest this node holds (A-484a) + haveOnly bool // heard no running event: only its have names the build it runs (A-482) + wantSource string + wantKnown bool + }{ + {"its entry answers first", "running", "running", false, true, false, false, SourceEntry, true}, + {"the entry of the composition's build is the running build's", "other", "other", true, true, false, false, SourceEntry, true}, + {"a composition with no manifest keeps the trunk's manifest and its data", "", "running", true, true, false, false, SourceTrunk, true}, + {"a composition and no trunk answer: its build and its data alone", "", "running", true, false, false, false, SourceComposition, false}, + {"the trunk alone", "", "", false, true, false, false, SourceTrunk, true}, + {"its running statement's manifest comes before the trunk's", "", "running", true, true, true, false, SourceStatement, true}, + {"only its have names the build, whose entry is held (A-482)", "running", "", false, false, false, true, SourceEntry, true}, + {"none answers", "", "", false, false, false, false, "", false}, + } { + t.Run(tc.name, func(t *testing.T) { + env := newFakeBus().env() + env.Node = "n1" + var said []string + env.Say = func(s string) { said = append(said, s) } + if tc.composition != "" { + fp := map[string]string{"running": running, "other": other}[tc.composition] + env.Running = func() []compositions.Composed { + return []compositions.Composed{{Node: "n1", Module: "xorg", FP: fp, Data: tc.data}} + } + } + c := New(env) + c.mu.Lock() + c.assignments["n1"] = meshmoduleassigner.Assignments{Node: "n1", Modules: []meshmoduleassigner.PlacedModule{{Module: "xorg"}}} + c.heard = map[string]map[string]heardRun{"n1": {"xorg": {fingerprint: running}}} + if tc.haveOnly { + c.heard = map[string]map[string]heardRun{} + c.haveRuns["xorg"], c.haveFp["xorg"] = true, running + } + if tc.statement { + digest := "sha256:" + strings.Repeat("9", 64) + c.heard["n1"]["xorg"] = heardRun{fingerprint: running, manifest: digest} + c.manifests[digest] = []byte(body) + } + if tc.trunk { + var man map[string]any + _ = json.Unmarshal([]byte(body), &man) + c.trunkBuilds = map[string]trunkBuild{"xorg": {running: running, fp: other, manifest: man}} + } + c.mu.Unlock() + if tc.entry != "" { + c.holdEntry(t, "xorg", map[string]string{"running": running, "other": other}[tc.entry], body) + } + c.askTrunkBuilds()() + c.layModules() + a := c.runningManifest("n1", "xorg", "") + again := c.runningManifest("n1", "xorg", "") + if a.Source != tc.wantSource || a.Known != tc.wantKnown || (a.Known && len(a.Manifest.Provides) != 1) { + t.Fatalf("answered %q (known %v, provides %d), want %q (known %v)", a.Source, a.Known, len(a.Manifest.Provides), tc.wantSource, tc.wantKnown) + } + sources := 0 + for _, line := range said { + if strings.Contains(line, "runs a build whose manifest is read from "+tc.wantSource) { + sources++ + } + } + if tc.wantSource != "" && (sources != 1 || again.Said != a.Said) { + t.Errorf("said %q, want its source once", said) + } + if tc.composition != "" && a.Data != tc.data && a.Source != SourceEntry && a.Source != SourceStatement { + t.Errorf("the composition's data = %v, want %v", a.Data, tc.data) + } + if !tc.wantKnown && !strings.Contains(a.Why, SourceEntry) || !tc.wantKnown && !strings.Contains(a.Why, SourceStatement) || !tc.wantKnown && !strings.Contains(a.Why, SourceTrunk) { + t.Errorf("the words of an answer that reads no manifest are %q, want the three sources named", a.Why) + } + }) + } + + // A module counted as running here alone answers a held module's offer and what the module keeps (A-482): one wanted here that + // runs nothing and whose have names no build answers nothing, and HoldsData reads the manifest the one place read. + env := newFakeBus().env() + env.Node = "n1" + c := New(env) + c.holdEntry(t, "xorg", running, body) + c.mu.Lock() + c.wants["xorg"] = meshwants.WantsByWantNodeModuleValue{Fp: other} + c.haveFp["xorg"] = running + c.mu.Unlock() + if _, ok := c.runningFor("n1")("xorg"); ok { + t.Error("a module wanted here that runs nothing answered a manifest as running") + } + if data, known := c.HoldsData(context.Background(), "xorg"); data || !known { + t.Errorf("HoldsData = %v, %v, want a manifest read that declares no data", data, known) + } + c.mu.Lock() + c.haveRuns["xorg"] = true + c.mu.Unlock() + if man, ok := c.runningFor("n1")("xorg"); !ok || len(man.Provides) != 1 { + t.Error("a module its have says runs answered no manifest") + } + } + // A-461, A-461a, A-461b: another node's module whose running build has no entry is laid from the entry of the build the assignment // places for it on that node, or, where the placement names none, of its trunk build; with neither entry it stays as its name // alone, and this node's own module is never laid from a placement. diff --git a/internal/selfcompose/running.go b/internal/selfcompose/running.go index f703c233..902dfe54 100644 --- a/internal/selfcompose/running.go +++ b/internal/selfcompose/running.go @@ -49,7 +49,7 @@ type runningAnswer struct { RunFP string // NeedTrunk says the trunk manifest is the one source left and is not held yet: the composer asks it beside the pass. NeedTrunk bool - // Said is the line to say, once per module and source (principle 3); empty when it was said before. + // Said is the line saying the source, for sayOnceEach (once per module and source, principle 3); empty when none answered. Said string } @@ -70,6 +70,7 @@ func (c *Composer) runningFpsLocked(running []engcomp.Composed, node, module, hi return fps } add(hint) + add(c.haveFp[module]) // the build its have names (A-482) for _, r := range running { if r.Module == module { add(r.FP) @@ -162,15 +163,12 @@ func (c *Composer) runningLocked(running []engcomp.Composed, node, module, hint } } if a.Source != "" { - line := fmt.Sprintf("%s runs a build whose manifest is read from %s", module, a.Source) - key := "running-manifest:" + node + "/" + module + "/" + a.Source - if c.told == nil { - c.told = map[string]bool{} - } - if !c.told[key] { - c.told[key] = true - a.Said = line + // Said once per module and source by sayOnceEach, by whoever uses the answer (principle 3). + where := "" + if node != c.env.Node { + where = " on " + node } + a.Said = fmt.Sprintf("%s%s runs a build whose manifest is read from %s", module, where, a.Source) } return a } @@ -185,7 +183,7 @@ func (c *Composer) runningManifest(node, module, hint string) runningAnswer { a := c.runningLocked(running, node, module, hint) c.mu.Unlock() if a.Said != "" { - c.env.Say(a.Said) + c.sayOnceEach(a.Said) } return a } diff --git a/internal/selfcompose/running_test.go b/internal/selfcompose/running_test.go deleted file mode 100644 index 2eabb1f9..00000000 --- a/internal/selfcompose/running_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package selfcompose - -import ( - "encoding/json" - "strings" - "testing" - - "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "github.com/novox/mesh-host/internal/compositions" -) - -// A-484: the manifest of the build a module runs here is answered in one place, from the entry of the running build, else the -// composition this node runs of it, else the module's trunk manifest; it says its source once per module and source, and where -// none answers its words name the three. -func TestTheManifestOfTheRunningBuildIsReadInOnePlace(t *testing.T) { - running := meshcatalogclient.SourcePrefix2 + strings.Repeat("1", 64) - other := meshcatalogclient.SourcePrefix2 + strings.Repeat("2", 64) - body := `{"module":"xorg","version":"1","provides":[{"name":"x11-display"}]}` - for _, tc := range []struct { - name string - entry string // the entry held, by fingerprint ("running", "other") or none - composition string // the composition this node runs: its build ("running", "other") or none - data bool - trunk bool - statement bool // the running event names a manifest this node holds (A-484a) - wantSource string - wantKnown bool - }{ - {"its entry answers first", "running", "running", false, true, false, SourceEntry, true}, - {"the entry of the composition's build is the running build's", "other", "other", true, true, false, SourceEntry, true}, - {"a composition with no manifest keeps the trunk's manifest and its data", "", "running", true, true, false, SourceTrunk, true}, - {"a composition and no trunk answer: its build and its data alone", "", "running", true, false, false, SourceComposition, false}, - {"the trunk alone", "", "", false, true, false, SourceTrunk, true}, - {"its running statement's manifest comes before the trunk's", "", "running", true, true, true, SourceStatement, true}, - {"none answers", "", "", false, false, false, "", false}, - } { - t.Run(tc.name, func(t *testing.T) { - env := newFakeBus().env() - env.Node = "n1" - var said []string - env.Say = func(s string) { said = append(said, s) } - if tc.composition != "" { - fp := map[string]string{"running": running, "other": other}[tc.composition] - env.Running = func() []compositions.Composed { - return []compositions.Composed{{Node: "n1", Module: "xorg", FP: fp, Data: tc.data}} - } - } - c := New(env) - c.mu.Lock() - c.assignments["n1"] = meshmoduleassigner.Assignments{Node: "n1", Modules: []meshmoduleassigner.PlacedModule{{Module: "xorg"}}} - c.heard = map[string]map[string]heardRun{"n1": {"xorg": {fingerprint: running}}} - if tc.statement { - digest := "sha256:" + strings.Repeat("9", 64) - c.heard["n1"]["xorg"] = heardRun{fingerprint: running, manifest: digest} - c.manifests[digest] = []byte(body) - } - if tc.trunk { - var man map[string]any - _ = json.Unmarshal([]byte(body), &man) - c.trunkBuilds = map[string]trunkBuild{"xorg": {running: running, fp: other, manifest: man}} - } - c.mu.Unlock() - if tc.entry != "" { - c.holdEntry(t, "xorg", map[string]string{"running": running, "other": other}[tc.entry], body) - } - a := c.runningManifest("n1", "xorg", "") - again := c.runningManifest("n1", "xorg", "") - if a.Source != tc.wantSource || a.Known != tc.wantKnown || (a.Known && len(a.Manifest.Provides) != 1) { - t.Fatalf("answered %q (known %v, provides %d), want %q (known %v)", a.Source, a.Known, len(a.Manifest.Provides), tc.wantSource, tc.wantKnown) - } - if tc.wantSource != "" && (len(said) != 1 || again.Said != "" || !strings.Contains(said[0], tc.wantSource)) { - t.Errorf("said %q (and %q again), want its source once", said, again.Said) - } - if tc.composition != "" && a.Data != tc.data && a.Source != SourceEntry && a.Source != SourceStatement { - t.Errorf("the composition's data = %v, want %v", a.Data, tc.data) - } - if !tc.wantKnown && !strings.Contains(a.Why, SourceEntry) || !tc.wantKnown && !strings.Contains(a.Why, SourceStatement) || !tc.wantKnown && !strings.Contains(a.Why, SourceTrunk) { - t.Errorf("the words of an answer that reads no manifest are %q, want the three sources named", a.Why) - } - }) - } -} diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 4c9a23f7..7aadc3dc 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -182,7 +182,9 @@ type Composer struct { inOwn map[string]bool // haveRuns is, by module, whether this node's have of it names a build in the phase running or held: a module that // keeps running what it runs, though no running event says it (design 47 G A-482). - haveRuns map[string]bool + haveRuns map[string]bool + // haveFp is, by module, the build that have names, where haveRuns says it runs: one of the builds the one place reads (A-482, A-484). + haveFp map[string]string modules map[string]meshcatalog.ModulesByModuleValue told map[string]bool // lines said once each (sayOnceEach) values map[string]map[string]any // by module, this node's @@ -428,7 +430,7 @@ func New(env Env) *Composer { func newComposer(env Env) *Composer { return &Composer{env: env, wants: map[string]meshwants.WantsByWantNodeModuleValue{}, assignments: map[string]meshmoduleassigner.Assignments{}, assignmentKeys: meshmoduleassignerclient.NewKeys(), - records: map[string]nodeengine.NodeRecord{}, inOwn: map[string]bool{}, haveRuns: map[string]bool{}, modules: map[string]meshcatalog.ModulesByModuleValue{}, values: map[string]map[string]any{}, + records: map[string]nodeengine.NodeRecord{}, inOwn: map[string]bool{}, haveRuns: map[string]bool{}, haveFp: map[string]string{}, modules: map[string]meshcatalog.ModulesByModuleValue{}, values: map[string]map[string]any{}, manifests: map[string][]byte{}, archives: map[string][]byte{}, handed: map[string]string{}, notTaken: map[string]string{}, unmadeSaid: map[string]string{}, rehand: make(chan struct{}, 1), sealedOwn: map[string]ownSealed{}, pkgs: map[string]compositions.Composed{}, asked: make(chan chan Pass, 8)} } @@ -867,9 +869,10 @@ func (c *Composer) takeModuleHave(u agent.Update) bool { if err != nil || isNode || node != c.env.Node { return false } - in, runs := false, false + in, runs, haveFp := false, false, "" if !u.Deleted { if h, err := nodeengineclient.DecodeHave(u.Value); err == nil { + haveFp = h.Fp runs = (h.Phase == nodeengineclient.PhaseRunning || h.Phase == nodeengineclient.PhaseHeld) && h.Fp != "" // A have that names nothing it runs, failed (the node could not compose a module it never ran: NotComposed), is // no module run inside the node's own composition: taken as one, it was composed from its manifest past its @@ -888,8 +891,10 @@ func (c *Composer) takeModuleHave(u agent.Update) bool { changed := c.haveRuns[module] != runs if runs { c.haveRuns[module] = true + c.haveFp[module] = haveFp } else { delete(c.haveRuns, module) + delete(c.haveFp, module) } if c.inOwn[module] == in { return changed @@ -1414,25 +1419,15 @@ func (c *Composer) Policy(ctx context.Context, module string) string { return policy } -// HoldsData says whether a module's manifest declares data (catalogue.Manifest.DataItems), as a composition's Data says -// it, and whether the manifest could be read: the agent's Data for what runs of a module from the kept declaration +// HoldsData says whether the manifest of the build a module runs here declares data (catalogue.Manifest.DataItems), as a +// composition's Data says it, and whether that manifest could be read, from the one place (A-484): the agent's Data for what runs of a module from the kept declaration // (agent.Env.HoldsData, issue 924). -func (c *Composer) HoldsData(ctx context.Context, module string) (data, known bool) { - c.mu.Lock() - m, ok := c.modules[module] - c.mu.Unlock() - if !ok || m.Manifest == "" { +func (c *Composer) HoldsData(_ context.Context, module string) (data, known bool) { + a := c.runningManifest(c.env.Node, module, "") + if !a.Known { return false, false } - raw, err := reader{c: c}.Manifest(ctx, m.Manifest) - if err != nil { - return false, false - } - parsed, held, err := c.taken(module, m, raw) - if err != nil || held != "" { - return false, false - } - return len(parsed.DataItems()) > 0, true + return len(a.Manifest.DataItems()) > 0, true } // taken is a module's manifest as this engine reads it (novox/hq ADR 0578 ยง3): decoded leniently, judging nothing, with