From f98d5ec748f6198640a11515a4da278fe0bf6dff Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 15:28:29 +0100 Subject: [PATCH] assigner: compose a machine across every watched branch (H-A-143) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The composition step judged a machine at the step's one branch revision, so an app registered on another watched branch was missing and the machine failed "nothing provides", ending every mover there "could not run". Each module's entry now comes from the newest revision of the first of its watched branches, in its registration's order, that lists it; the candidate's revision replaces its branch's newest for the modules it moves. A module assigned to the machine that no watched branch lists is said in the machine's reason. Builds: design 47 H H-A-143, G §532 --- .../mesh-delivery-assigner/compositionrun.go | 119 +++++++++++++++++- .../compositionrun_test.go | 112 +++++++++++++++-- .../mesh-delivery-assigner/compositionruns.go | 2 +- .../compositionruns_test.go | 2 +- .../cmd/mesh-delivery-assigner/halves.go | 18 +++ .../cmd/mesh-delivery-assigner/main.go | 2 +- .../cmd/mesh-delivery-assigner/world.go | 9 ++ 7 files changed, 248 insertions(+), 16 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go index 2482ebc0..266aaf2d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "slices" "strings" "sync" @@ -53,6 +54,13 @@ type compositionRun struct { call func(key string, body any) (json.RawMessage, error) // trunk is the trunk's catalog half as the assigner holds it. trunk func() (mcc.Half, bool) + // newest is the newest catalog half the assigner's views hold of each watched branch (halves.views), and watched a module's + // watched branches in its registration's order (halves.watchedOf): a machine's composition takes each module's manifest across + // them (design 47 H-A-143, G §532). Nil: the step's revision alone. + newest func() map[string]mcc.Half + watched func(module string) []string + // assigned is the modules the assignments place on a machine (world.assignments), for the modules no watched branch lists. + assigned func(node string) []string // running is the last `running` heard of a module on a node (world.runningOf), never the node-engine's bucket. running func(node, module string) (ne.Running, bool) // away says the watchman holds machine away (machine-away, not back since): the one presence the assigner keeps (world.awayNow), @@ -123,7 +131,7 @@ func (r *compositionRun) read(ctx context.Context, module string, keys []string, runs := r.runs.await(ctx, keys) setPhase(ctx, meshwants.StepProgressEventPhaseReport) for _, key := range keys { - ends = append(ends, judgeRun(module, runs[key])) + ends = append(ends, judgeRun(module, runs[key], r.unlistedOn(runs[key].Machine))) } out := outcomeOf(ends) out.Shared = r.runs.sharedOf(keys, runs) @@ -209,6 +217,9 @@ func (p *runPrep) get(ctx context.Context) error { p.err = errors.New("the step's catalog revision is not read: " + err.Error()) return } + if p.r.newest != nil && p.r.watched != nil { + half = acrossBranches(p.step, half, p.r.newest(), p.r.watched) + } if p.seats, p.err = p.r.seats(ctx, half.Revision.Definitions); p.err != nil { return } @@ -289,7 +300,10 @@ const notTried = "not tried: the search reached its bound" // could not run where the module was not tried (A-470b); a machine that does not compose and blames no module fails every step on // it, on the node's own account; a verdict with no `blamed` (a composer before §721) is judged as before, with the warning "the // composer names no culprits"; otherwise the machine passes, with a warning for each other module blamed there. -func judgeRun(module string, run meshwants.CompositionRun) machineEnd { +// +// unlisted are the modules the machine is assigned that no watched branch lists: each is said in the machine's reason, or as a +// warning where it composes, never dropped in silence (H-A-143). +func judgeRun(module string, run meshwants.CompositionRun, unlisted []string) machineEnd { node := run.Machine if run.Ended == nil { return machineEnd{couldNot: node + ": the step stopped before its composition run " + runKeyOf(run.Candidate, node) + " ended"} @@ -305,6 +319,14 @@ func judgeRun(module string, run meshwants.CompositionRun) machineEnd { return machineEnd{couldNot: node + " was not judged: " + m.NotJudged} } why := strings.Join(appendSome(append(slices.Clone(m.Problems), m.Refusals...), m.Reason), "; ") + var notListed []string + for _, u := range unlisted { + notListed = append(notListed, node+" is assigned "+u+", which no watched branch lists (H-A-143)") + } + if !m.Composes && len(notListed) > 0 { + why = strings.Join(appendSome(notListed, why), "; ") + notListed = nil + } switch { case m.Blamed == nil && !m.Composes: // an old composer (§721) return machineEnd{judged: true, failed: node + " does not compose: " + why, warnings: []string{node + ": the composer names no culprits"}} @@ -326,7 +348,7 @@ func judgeRun(module string, run meshwants.CompositionRun) machineEnd { if len(own) > 0 { end.failed, end.couldNot = node+" holds "+module+": "+strings.Join(own, "; "), "" } - end.warnings = others + end.warnings = append(others, notListed...) return end } @@ -503,6 +525,76 @@ func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]string, err return manifests, none, nil } +// acrossBranches is the catalog half a machine's composition reads (design 47 H-A-143, G §532): each module's entry from the newest +// revision of the first of its watched branches, in its registration's order, that lists it; the candidate's revision replaces its +// branch's newest for the modules it moves (its movers and the step's module), and the whole of it where it is that branch's newest +// or the branch's is not held. The half keeps the candidate's revision, its definitions with it. +func acrossBranches(step meshwants.StepRun, candidate mcc.Half, newest map[string]mcc.Half, watched func(module string) []string) mcc.Half { + halves := make(map[string]mcc.Half, len(newest)+1) + for b, h := range newest { + halves[b] = h + } + if own, held := newest[step.Branch]; !held || candidate.Revision.Revision >= own.Revision.Revision { + halves[step.Branch] = candidate + } else { + mix := mcc.Half{Revision: own.Revision, Entries: maps.Clone(own.Entries)} + moved := []string{step.Module} + for _, m := range candidate.Revision.Movers { + moved = append(moved, m.Module) + } + for _, m := range moved { + if e, ok := candidate.Entries[m]; ok { + mix.Entries[m] = e + } else { + delete(mix.Entries, m) + } + } + halves[step.Branch] = mix + } + out := mcc.Half{Revision: candidate.Revision, Entries: map[string]meshcatalog.CatalogEntry{}} + for _, h := range halves { + for name := range h.Entries { + if _, done := out.Entries[name]; done { + continue + } + order := watched(name) + if len(order) == 0 { // no branch is followed yet: the step's own + order = []string{step.Branch} + } + for _, b := range order { + if e, ok := halves[b].Entries[name]; ok { + out.Entries[name] = e + break + } + } + } + } + return out +} + +// unlistedOn is the modules the assignments place on node that no watched branch's newest revision lists (H-A-143), by name. +func (r *compositionRun) unlistedOn(node string) []string { + if r.assigned == nil || r.newest == nil { + return nil + } + newest := r.newest() + var out []string + for _, m := range r.assigned(node) { + listed := false + for _, h := range newest { + if _, ok := h.Entries[m]; ok { + listed = true + break + } + } + if !listed { + out = append(out, m) + } + } + slices.Sort(out) + return slices.Compact(out) +} + // nodeEngineEntry is the entry of the module that claims the seat node-engine, or false. func nodeEngineEntry(entries map[string]meshcatalog.CatalogEntry) (meshcatalog.CatalogEntry, bool) { for _, name := range slices.Sorted(mapsKeys(entries)) { @@ -616,3 +708,24 @@ func runAskPublisher(js jetstream.JetStream) func(ctx context.Context, a buildst return err } } + +// newestOf is the newest catalog half the views hold of each watched branch, or nil while the catalog half is not followed. +func (iv *catalogViews) newestOf() func() map[string]mcc.Half { + if iv == nil { + return nil + } + return func() map[string]mcc.Half { + if iv.h == nil { + return map[string]mcc.Half{} + } + return iv.h.views() + } +} + +// watchedOf is a module's watched branches in its registration's order (halves.watchedOf); none while the half is not followed. +func (iv *catalogViews) watchedOf(module string) []string { + if iv == nil || iv.h == nil { + return nil + } + return iv.h.watchedOf(module) +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go index ed48790f..2f4c0df9 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "maps" "slices" "strings" "sync" @@ -259,6 +260,23 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) { } } + // Another watched branch's app is in every machine's case, and a module assigned there that no watched branch lists is said in + // the machine's reason (H-A-143, §532). + across := newRunStand(t, "src2:e1", "src2:e1", "d1") + across.r.newest = func() map[string]mcc.Half { + return map[string]mcc.Half{"master": {Revision: meshcatalog.CatalogRevision{Branch: "master", Revision: 2}, + Entries: map[string]meshcatalog.CatalogEntry{"f1-circuits": {Module: "f1-circuits", Fp: "f1", Manifest: map[string]any{"module": "f1-circuits"}}}}} + } + across.r.watched = func(module string) []string { return []string{"main", "master"} } + across.r.assigned = func(node string) []string { return []string{"m", "f1-circuits", "checker"} } + if out := across.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed || + !strings.Contains(out.Said, "g14 is assigned checker, which no watched branch lists (H-A-143)") { + t.Errorf("the step across the watched branches came to %+v", out) + } + if c := across.requests["ace"].Cases[0]; string(c.Manifests["f1-circuits"].Raw) != `{"module":"f1-circuits"}` || len(c.Manifests) != 5 { + t.Errorf("the case across the watched branches holds %v", slices.Sorted(maps.Keys(c.Manifests))) + } + // At the inputs state's cap the ask is refused, naming the store (§354). full := newRunStand(t, "src2:e1", "src2:e1", "d1") full.r.keeper.used = func() (int64, error) { return inputsCap, nil } @@ -438,21 +456,27 @@ func TestAMachineIsJudgedByTheModulesItBlames(t *testing.T) { run meshwants.CompositionRun failed, couldNo string warnings []string + unlisted []string }{ - {"blamed module fails", "a", runOf(blamesA), "ace holds a: /etc/x is a's and c's", "", nil}, - {"another module passes with a warning", "b", runOf(blamesA), "", "", []string{"ace holds a: /etc/x is a's and c's"}}, - {"blames no module fails both", "b", runOf(`{"composes":false,"reason":"the node's own composition","blamed":[]}`), "ace does not compose and blames no module: the node's own composition", "", nil}, - {"an old composer", "b", runOf(`{"composes":false,"reason":"x"}`), "ace does not compose: x", "", []string{"ace: the composer names no culprits"}}, - {"composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "", nil}, + {"blamed module fails", "a", runOf(blamesA), "ace holds a: /etc/x is a's and c's", "", nil, nil}, + {"another module passes with a warning", "b", runOf(blamesA), "", "", []string{"ace holds a: /etc/x is a's and c's"}, nil}, + {"blames no module fails both", "b", runOf(`{"composes":false,"reason":"the node's own composition","blamed":[]}`), "ace does not compose and blames no module: the node's own composition", "", nil, nil}, + {"an old composer", "b", runOf(`{"composes":false,"reason":"x"}`), "ace does not compose: x", "", []string{"ace: the composer names no culprits"}, nil}, + {"composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "", nil, nil}, {"own module not tried", "a", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "", - "ace holds a untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", nil}, + "ace holds a untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", nil, nil}, {"another not tried", "b", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "", "", - []string{"ace holds a: not tried: the search reached its bound"}}, - {"not judged", "a", runOf(`{"composes":false,"not_judged":"no seat list","blamed":[]}`), "", "ace was not judged: no seat list", nil}, - {"could not run", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"}, "", "ace: gone", nil}, + []string{"ace holds a: not tried: the search reached its bound"}, nil}, + {"not judged", "a", runOf(`{"composes":false,"not_judged":"no seat list","blamed":[]}`), "", "ace was not judged: no seat list", nil, nil}, + {"could not run", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"}, "", "ace: gone", nil, nil}, + // H-A-143: a module assigned there that no watched branch lists is said in the reason, or as a warning where it composes. + {"an unlisted module in the reason", "b", runOf(`{"composes":false,"reason":"nothing provides \"photos\"","blamed":[]}`), + "ace does not compose and blames no module: ace is assigned photos, which no watched branch lists (H-A-143); nothing provides \"photos\"", "", nil, []string{"photos"}}, + {"an unlisted module where it composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "", + []string{"ace is assigned photos, which no watched branch lists (H-A-143)"}, []string{"photos"}}, } { t.Run(tc.name, func(t *testing.T) { - e := judgeRun(tc.module, tc.run) + e := judgeRun(tc.module, tc.run, tc.unlisted) if e.failed != tc.failed || e.couldNot != tc.couldNo || !slices.Equal(e.warnings, tc.warnings) { t.Errorf("judged %+v, want failed %q, could not run %q, warnings %q", e, tc.failed, tc.couldNo, tc.warnings) } @@ -572,3 +596,71 @@ func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) { } } } + +// Design 47 H-A-143, G §532: a machine's composition takes each module's entry from the newest revision of the first of its +// watched branches, in its registration's order, that lists it; the candidate's revision replaces its branch's newest for the +// modules it moves, and whole where it is that branch's newest; the half keeps the candidate's revision. +func TestACompositionTakesEachModuleAcrossTheWatchedBranches(t *testing.T) { + entry := func(module, fp string) meshcatalog.CatalogEntry { + return meshcatalog.CatalogEntry{Module: module, Fp: fp, Manifest: map[string]any{"module": module}} + } + half := func(branch string, revision int64, movers []string, entries ...meshcatalog.CatalogEntry) mcc.Half { + h := mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: branch, Revision: revision}, Entries: map[string]meshcatalog.CatalogEntry{}} + for _, m := range movers { + h.Revision.Movers = append(h.Revision.Movers, meshcatalog.TakeInEntry{Module: m}) + } + for _, e := range entries { + h.Entries[e.Module] = e + } + return h + } + newest := map[string]mcc.Half{ + "main": half("main", 4, nil, entry("engine", "e1"), entry("a", "a1"), entry("both", "main")), + "master": half("master", 9, nil, entry("f1-circuits", "f1")), + "nox-mesh": half("nox-mesh", 3, nil, entry("link2pay", "l1"), entry("photos", "p1"), entry("both", "nox")), + } + registrations := map[string][]string{"both": {"nox-mesh", "main"}, "f1-circuits": {"master"}, "link2pay": {"nox-mesh"}, "photos": {"nox-mesh"}} + watched := func(module string) []string { + if b, ok := registrations[module]; ok { + return b + } + return []string{"main", "master", "nox-mesh"} + } + for _, tc := range []struct { + name string + step meshwants.StepRun + candidate mcc.Half + newest map[string]mcc.Half + want map[string]string // module -> fingerprint + }{ + {"the newest of main, the others' apps beside it", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5}, + half("main", 5, []string{"a"}, entry("engine", "e1"), entry("a", "a2"), entry("both", "main")), newest, + map[string]string{"engine": "e1", "a": "a2", "both": "nox", "f1-circuits": "f1", "link2pay": "l1", "photos": "p1"}}, + {"an older candidate moves only its movers", meshwants.StepRun{Branch: "nox-mesh", Module: "link2pay", Revision: 2}, + half("nox-mesh", 2, []string{"photos"}, entry("link2pay", "l0"), entry("both", "nox0")), newest, + map[string]string{"engine": "e1", "a": "a1", "both": "nox", "f1-circuits": "f1", "link2pay": "l0"}}, + {"a branch not yet held is the candidate's", meshwants.StepRun{Branch: "master", Module: "f1-circuits", Revision: 1}, + half("master", 1, nil, entry("f1-circuits", "f0")), map[string]mcc.Half{"main": newest["main"]}, + map[string]string{"engine": "e1", "a": "a1", "both": "main", "f1-circuits": "f0"}}, + {"no view held yet: the candidate alone", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5}, + half("main", 5, nil, entry("a", "a2")), nil, map[string]string{"a": "a2"}}, + } { + t.Run(tc.name, func(t *testing.T) { + w := watched + if tc.newest == nil { + w = func(string) []string { return nil } + } + got := acrossBranches(tc.step, tc.candidate, tc.newest, w) + fps := map[string]string{} + for name, e := range got.Entries { + fps[name] = e.Fp + } + if !maps.Equal(fps, tc.want) { + t.Errorf("composed from %v, want %v", fps, tc.want) + } + if got.Revision.Revision != tc.candidate.Revision.Revision { + t.Errorf("the half keeps revision %d, want the candidate's %d", got.Revision.Revision, tc.candidate.Revision.Revision) + } + }) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go index 37f2a774..f8844555 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go @@ -818,7 +818,7 @@ func (r *compositionRuns) leave(pipeline string, stop func(ids []string)) ([]str // passedOn is whether the step of module passed on the machine of an ended run (§722): neither failed nor could not run there. func passedOn(module string, run meshwants.CompositionRun) bool { - e := judgeRun(module, run) + e := judgeRun(module, run, nil) // the unlisted modules change only the words, never whether it passed return e.failed == "" && e.couldNot == "" } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go index 7a0ccf15..d0be3c0d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go @@ -416,7 +416,7 @@ func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) { len(*drops) != 1 || !strings.HasPrefix((*drops)[0], "run-1: ") { t.Fatalf("past the bound: %+v, drops %v", run, *drops) } - if got := judgeRun("a", run); got.couldNot != run.Said { + if got := judgeRun("a", run, nil); got.couldNot != run.Said { t.Errorf("a reader of the run past its bound: %+v", got) } late := buildstep.StepEnded{Ask: "run-1", Asker: runAsker, Outcome: buildstep.OutcomePassed, Output: "{}"} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go index 49f8a57c..404dc558 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go @@ -191,3 +191,21 @@ func (h *halves) views() map[string]mcc.Half { } return out } + +// watchedOf is module's watched branches in its registration's order (the branch `register` named first, then each branch in the +// order `watch` added it), as the follower's registrations hold it; before its registration is held, the branches followed, in +// the order they came, the gate's source's before the first registrations (design 47 G §532, H-A-143). +func (h *halves) watchedOf(module string) []string { + h.mu.Lock() + b := h.branches + h.mu.Unlock() + if b == nil { + return nil + } + if b.Registrations != nil { + if reg, ok := b.Registrations.Registration(module); ok && len(reg.Branches) > 0 { + return slices.Clone(reg.Branches) + } + } + return b.Followed() +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 745d4e0a..e4fac8b3 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -665,7 +665,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, // The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine. if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil { run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, - trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf} + trunk: trunkOf(iv, w.trunkName), newest: iv.newestOf(), watched: iv.watchedOf, assigned: w.assignedOn, running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf} steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume, validate: run.validate, resumeValidate: run.resumeValidate, ended: runs.stepEnded, now: time.Now, logf: logger.Printf} move := steps.move(ctx) // the readers a rerun carried (§727, A-470d) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index 9c0cf124..bb82e620 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -553,3 +553,12 @@ func (w *world) placedBranches() map[string]map[string]string { } return out } + +// assignedOn is the modules the assignments place on node. +func (w *world) assignedOn(node string) []string { + var out []string + for _, m := range w.assignments()[node].Modules { + out = append(out, m.Module) + } + return out +}