diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go index 2482ebc0..266aaf2d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "slices" "strings" "sync" @@ -53,6 +54,13 @@ type compositionRun struct { call func(key string, body any) (json.RawMessage, error) // trunk is the trunk's catalog half as the assigner holds it. trunk func() (mcc.Half, bool) + // newest is the newest catalog half the assigner's views hold of each watched branch (halves.views), and watched a module's + // watched branches in its registration's order (halves.watchedOf): a machine's composition takes each module's manifest across + // them (design 47 H-A-143, G §532). Nil: the step's revision alone. + newest func() map[string]mcc.Half + watched func(module string) []string + // assigned is the modules the assignments place on a machine (world.assignments), for the modules no watched branch lists. + assigned func(node string) []string // running is the last `running` heard of a module on a node (world.runningOf), never the node-engine's bucket. running func(node, module string) (ne.Running, bool) // away says the watchman holds machine away (machine-away, not back since): the one presence the assigner keeps (world.awayNow), @@ -123,7 +131,7 @@ func (r *compositionRun) read(ctx context.Context, module string, keys []string, runs := r.runs.await(ctx, keys) setPhase(ctx, meshwants.StepProgressEventPhaseReport) for _, key := range keys { - ends = append(ends, judgeRun(module, runs[key])) + ends = append(ends, judgeRun(module, runs[key], r.unlistedOn(runs[key].Machine))) } out := outcomeOf(ends) out.Shared = r.runs.sharedOf(keys, runs) @@ -209,6 +217,9 @@ func (p *runPrep) get(ctx context.Context) error { p.err = errors.New("the step's catalog revision is not read: " + err.Error()) return } + if p.r.newest != nil && p.r.watched != nil { + half = acrossBranches(p.step, half, p.r.newest(), p.r.watched) + } if p.seats, p.err = p.r.seats(ctx, half.Revision.Definitions); p.err != nil { return } @@ -289,7 +300,10 @@ const notTried = "not tried: the search reached its bound" // could not run where the module was not tried (A-470b); a machine that does not compose and blames no module fails every step on // it, on the node's own account; a verdict with no `blamed` (a composer before §721) is judged as before, with the warning "the // composer names no culprits"; otherwise the machine passes, with a warning for each other module blamed there. -func judgeRun(module string, run meshwants.CompositionRun) machineEnd { +// +// unlisted are the modules the machine is assigned that no watched branch lists: each is said in the machine's reason, or as a +// warning where it composes, never dropped in silence (H-A-143). +func judgeRun(module string, run meshwants.CompositionRun, unlisted []string) machineEnd { node := run.Machine if run.Ended == nil { return machineEnd{couldNot: node + ": the step stopped before its composition run " + runKeyOf(run.Candidate, node) + " ended"} @@ -305,6 +319,14 @@ func judgeRun(module string, run meshwants.CompositionRun) machineEnd { return machineEnd{couldNot: node + " was not judged: " + m.NotJudged} } why := strings.Join(appendSome(append(slices.Clone(m.Problems), m.Refusals...), m.Reason), "; ") + var notListed []string + for _, u := range unlisted { + notListed = append(notListed, node+" is assigned "+u+", which no watched branch lists (H-A-143)") + } + if !m.Composes && len(notListed) > 0 { + why = strings.Join(appendSome(notListed, why), "; ") + notListed = nil + } switch { case m.Blamed == nil && !m.Composes: // an old composer (§721) return machineEnd{judged: true, failed: node + " does not compose: " + why, warnings: []string{node + ": the composer names no culprits"}} @@ -326,7 +348,7 @@ func judgeRun(module string, run meshwants.CompositionRun) machineEnd { if len(own) > 0 { end.failed, end.couldNot = node+" holds "+module+": "+strings.Join(own, "; "), "" } - end.warnings = others + end.warnings = append(others, notListed...) return end } @@ -503,6 +525,76 @@ func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]string, err return manifests, none, nil } +// acrossBranches is the catalog half a machine's composition reads (design 47 H-A-143, G §532): each module's entry from the newest +// revision of the first of its watched branches, in its registration's order, that lists it; the candidate's revision replaces its +// branch's newest for the modules it moves (its movers and the step's module), and the whole of it where it is that branch's newest +// or the branch's is not held. The half keeps the candidate's revision, its definitions with it. +func acrossBranches(step meshwants.StepRun, candidate mcc.Half, newest map[string]mcc.Half, watched func(module string) []string) mcc.Half { + halves := make(map[string]mcc.Half, len(newest)+1) + for b, h := range newest { + halves[b] = h + } + if own, held := newest[step.Branch]; !held || candidate.Revision.Revision >= own.Revision.Revision { + halves[step.Branch] = candidate + } else { + mix := mcc.Half{Revision: own.Revision, Entries: maps.Clone(own.Entries)} + moved := []string{step.Module} + for _, m := range candidate.Revision.Movers { + moved = append(moved, m.Module) + } + for _, m := range moved { + if e, ok := candidate.Entries[m]; ok { + mix.Entries[m] = e + } else { + delete(mix.Entries, m) + } + } + halves[step.Branch] = mix + } + out := mcc.Half{Revision: candidate.Revision, Entries: map[string]meshcatalog.CatalogEntry{}} + for _, h := range halves { + for name := range h.Entries { + if _, done := out.Entries[name]; done { + continue + } + order := watched(name) + if len(order) == 0 { // no branch is followed yet: the step's own + order = []string{step.Branch} + } + for _, b := range order { + if e, ok := halves[b].Entries[name]; ok { + out.Entries[name] = e + break + } + } + } + } + return out +} + +// unlistedOn is the modules the assignments place on node that no watched branch's newest revision lists (H-A-143), by name. +func (r *compositionRun) unlistedOn(node string) []string { + if r.assigned == nil || r.newest == nil { + return nil + } + newest := r.newest() + var out []string + for _, m := range r.assigned(node) { + listed := false + for _, h := range newest { + if _, ok := h.Entries[m]; ok { + listed = true + break + } + } + if !listed { + out = append(out, m) + } + } + slices.Sort(out) + return slices.Compact(out) +} + // nodeEngineEntry is the entry of the module that claims the seat node-engine, or false. func nodeEngineEntry(entries map[string]meshcatalog.CatalogEntry) (meshcatalog.CatalogEntry, bool) { for _, name := range slices.Sorted(mapsKeys(entries)) { @@ -616,3 +708,24 @@ func runAskPublisher(js jetstream.JetStream) func(ctx context.Context, a buildst return err } } + +// newestOf is the newest catalog half the views hold of each watched branch, or nil while the catalog half is not followed. +func (iv *catalogViews) newestOf() func() map[string]mcc.Half { + if iv == nil { + return nil + } + return func() map[string]mcc.Half { + if iv.h == nil { + return map[string]mcc.Half{} + } + return iv.h.views() + } +} + +// watchedOf is a module's watched branches in its registration's order (halves.watchedOf); none while the half is not followed. +func (iv *catalogViews) watchedOf(module string) []string { + if iv == nil || iv.h == nil { + return nil + } + return iv.h.watchedOf(module) +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go index ed48790f..2f4c0df9 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "maps" "slices" "strings" "sync" @@ -259,6 +260,23 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) { } } + // Another watched branch's app is in every machine's case, and a module assigned there that no watched branch lists is said in + // the machine's reason (H-A-143, §532). + across := newRunStand(t, "src2:e1", "src2:e1", "d1") + across.r.newest = func() map[string]mcc.Half { + return map[string]mcc.Half{"master": {Revision: meshcatalog.CatalogRevision{Branch: "master", Revision: 2}, + Entries: map[string]meshcatalog.CatalogEntry{"f1-circuits": {Module: "f1-circuits", Fp: "f1", Manifest: map[string]any{"module": "f1-circuits"}}}}} + } + across.r.watched = func(module string) []string { return []string{"main", "master"} } + across.r.assigned = func(node string) []string { return []string{"m", "f1-circuits", "checker"} } + if out := across.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed || + !strings.Contains(out.Said, "g14 is assigned checker, which no watched branch lists (H-A-143)") { + t.Errorf("the step across the watched branches came to %+v", out) + } + if c := across.requests["ace"].Cases[0]; string(c.Manifests["f1-circuits"].Raw) != `{"module":"f1-circuits"}` || len(c.Manifests) != 5 { + t.Errorf("the case across the watched branches holds %v", slices.Sorted(maps.Keys(c.Manifests))) + } + // At the inputs state's cap the ask is refused, naming the store (§354). full := newRunStand(t, "src2:e1", "src2:e1", "d1") full.r.keeper.used = func() (int64, error) { return inputsCap, nil } @@ -438,21 +456,27 @@ func TestAMachineIsJudgedByTheModulesItBlames(t *testing.T) { run meshwants.CompositionRun failed, couldNo string warnings []string + unlisted []string }{ - {"blamed module fails", "a", runOf(blamesA), "ace holds a: /etc/x is a's and c's", "", nil}, - {"another module passes with a warning", "b", runOf(blamesA), "", "", []string{"ace holds a: /etc/x is a's and c's"}}, - {"blames no module fails both", "b", runOf(`{"composes":false,"reason":"the node's own composition","blamed":[]}`), "ace does not compose and blames no module: the node's own composition", "", nil}, - {"an old composer", "b", runOf(`{"composes":false,"reason":"x"}`), "ace does not compose: x", "", []string{"ace: the composer names no culprits"}}, - {"composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "", nil}, + {"blamed module fails", "a", runOf(blamesA), "ace holds a: /etc/x is a's and c's", "", nil, nil}, + {"another module passes with a warning", "b", runOf(blamesA), "", "", []string{"ace holds a: /etc/x is a's and c's"}, nil}, + {"blames no module fails both", "b", runOf(`{"composes":false,"reason":"the node's own composition","blamed":[]}`), "ace does not compose and blames no module: the node's own composition", "", nil, nil}, + {"an old composer", "b", runOf(`{"composes":false,"reason":"x"}`), "ace does not compose: x", "", []string{"ace: the composer names no culprits"}, nil}, + {"composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "", nil, nil}, {"own module not tried", "a", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "", - "ace holds a untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", nil}, + "ace holds a untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", nil, nil}, {"another not tried", "b", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "", "", - []string{"ace holds a: not tried: the search reached its bound"}}, - {"not judged", "a", runOf(`{"composes":false,"not_judged":"no seat list","blamed":[]}`), "", "ace was not judged: no seat list", nil}, - {"could not run", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"}, "", "ace: gone", nil}, + []string{"ace holds a: not tried: the search reached its bound"}, nil}, + {"not judged", "a", runOf(`{"composes":false,"not_judged":"no seat list","blamed":[]}`), "", "ace was not judged: no seat list", nil, nil}, + {"could not run", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"}, "", "ace: gone", nil, nil}, + // H-A-143: a module assigned there that no watched branch lists is said in the reason, or as a warning where it composes. + {"an unlisted module in the reason", "b", runOf(`{"composes":false,"reason":"nothing provides \"photos\"","blamed":[]}`), + "ace does not compose and blames no module: ace is assigned photos, which no watched branch lists (H-A-143); nothing provides \"photos\"", "", nil, []string{"photos"}}, + {"an unlisted module where it composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "", + []string{"ace is assigned photos, which no watched branch lists (H-A-143)"}, []string{"photos"}}, } { t.Run(tc.name, func(t *testing.T) { - e := judgeRun(tc.module, tc.run) + e := judgeRun(tc.module, tc.run, tc.unlisted) if e.failed != tc.failed || e.couldNot != tc.couldNo || !slices.Equal(e.warnings, tc.warnings) { t.Errorf("judged %+v, want failed %q, could not run %q, warnings %q", e, tc.failed, tc.couldNo, tc.warnings) } @@ -572,3 +596,71 @@ func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) { } } } + +// Design 47 H-A-143, G §532: a machine's composition takes each module's entry from the newest revision of the first of its +// watched branches, in its registration's order, that lists it; the candidate's revision replaces its branch's newest for the +// modules it moves, and whole where it is that branch's newest; the half keeps the candidate's revision. +func TestACompositionTakesEachModuleAcrossTheWatchedBranches(t *testing.T) { + entry := func(module, fp string) meshcatalog.CatalogEntry { + return meshcatalog.CatalogEntry{Module: module, Fp: fp, Manifest: map[string]any{"module": module}} + } + half := func(branch string, revision int64, movers []string, entries ...meshcatalog.CatalogEntry) mcc.Half { + h := mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: branch, Revision: revision}, Entries: map[string]meshcatalog.CatalogEntry{}} + for _, m := range movers { + h.Revision.Movers = append(h.Revision.Movers, meshcatalog.TakeInEntry{Module: m}) + } + for _, e := range entries { + h.Entries[e.Module] = e + } + return h + } + newest := map[string]mcc.Half{ + "main": half("main", 4, nil, entry("engine", "e1"), entry("a", "a1"), entry("both", "main")), + "master": half("master", 9, nil, entry("f1-circuits", "f1")), + "nox-mesh": half("nox-mesh", 3, nil, entry("link2pay", "l1"), entry("photos", "p1"), entry("both", "nox")), + } + registrations := map[string][]string{"both": {"nox-mesh", "main"}, "f1-circuits": {"master"}, "link2pay": {"nox-mesh"}, "photos": {"nox-mesh"}} + watched := func(module string) []string { + if b, ok := registrations[module]; ok { + return b + } + return []string{"main", "master", "nox-mesh"} + } + for _, tc := range []struct { + name string + step meshwants.StepRun + candidate mcc.Half + newest map[string]mcc.Half + want map[string]string // module -> fingerprint + }{ + {"the newest of main, the others' apps beside it", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5}, + half("main", 5, []string{"a"}, entry("engine", "e1"), entry("a", "a2"), entry("both", "main")), newest, + map[string]string{"engine": "e1", "a": "a2", "both": "nox", "f1-circuits": "f1", "link2pay": "l1", "photos": "p1"}}, + {"an older candidate moves only its movers", meshwants.StepRun{Branch: "nox-mesh", Module: "link2pay", Revision: 2}, + half("nox-mesh", 2, []string{"photos"}, entry("link2pay", "l0"), entry("both", "nox0")), newest, + map[string]string{"engine": "e1", "a": "a1", "both": "nox", "f1-circuits": "f1", "link2pay": "l0"}}, + {"a branch not yet held is the candidate's", meshwants.StepRun{Branch: "master", Module: "f1-circuits", Revision: 1}, + half("master", 1, nil, entry("f1-circuits", "f0")), map[string]mcc.Half{"main": newest["main"]}, + map[string]string{"engine": "e1", "a": "a1", "both": "main", "f1-circuits": "f0"}}, + {"no view held yet: the candidate alone", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5}, + half("main", 5, nil, entry("a", "a2")), nil, map[string]string{"a": "a2"}}, + } { + t.Run(tc.name, func(t *testing.T) { + w := watched + if tc.newest == nil { + w = func(string) []string { return nil } + } + got := acrossBranches(tc.step, tc.candidate, tc.newest, w) + fps := map[string]string{} + for name, e := range got.Entries { + fps[name] = e.Fp + } + if !maps.Equal(fps, tc.want) { + t.Errorf("composed from %v, want %v", fps, tc.want) + } + if got.Revision.Revision != tc.candidate.Revision.Revision { + t.Errorf("the half keeps revision %d, want the candidate's %d", got.Revision.Revision, tc.candidate.Revision.Revision) + } + }) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go index 37f2a774..f8844555 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go @@ -818,7 +818,7 @@ func (r *compositionRuns) leave(pipeline string, stop func(ids []string)) ([]str // passedOn is whether the step of module passed on the machine of an ended run (§722): neither failed nor could not run there. func passedOn(module string, run meshwants.CompositionRun) bool { - e := judgeRun(module, run) + e := judgeRun(module, run, nil) // the unlisted modules change only the words, never whether it passed return e.failed == "" && e.couldNot == "" } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go index 7a0ccf15..d0be3c0d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go @@ -416,7 +416,7 @@ func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) { len(*drops) != 1 || !strings.HasPrefix((*drops)[0], "run-1: ") { t.Fatalf("past the bound: %+v, drops %v", run, *drops) } - if got := judgeRun("a", run); got.couldNot != run.Said { + if got := judgeRun("a", run, nil); got.couldNot != run.Said { t.Errorf("a reader of the run past its bound: %+v", got) } late := buildstep.StepEnded{Ask: "run-1", Asker: runAsker, Outcome: buildstep.OutcomePassed, Output: "{}"} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go index 49f8a57c..404dc558 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go @@ -191,3 +191,21 @@ func (h *halves) views() map[string]mcc.Half { } return out } + +// watchedOf is module's watched branches in its registration's order (the branch `register` named first, then each branch in the +// order `watch` added it), as the follower's registrations hold it; before its registration is held, the branches followed, in +// the order they came, the gate's source's before the first registrations (design 47 G §532, H-A-143). +func (h *halves) watchedOf(module string) []string { + h.mu.Lock() + b := h.branches + h.mu.Unlock() + if b == nil { + return nil + } + if b.Registrations != nil { + if reg, ok := b.Registrations.Registration(module); ok && len(reg.Branches) > 0 { + return slices.Clone(reg.Branches) + } + } + return b.Followed() +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 745d4e0a..e4fac8b3 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -665,7 +665,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, // The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine. if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil { run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, - trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf} + trunk: trunkOf(iv, w.trunkName), newest: iv.newestOf(), watched: iv.watchedOf, assigned: w.assignedOn, running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf} steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume, validate: run.validate, resumeValidate: run.resumeValidate, ended: runs.stepEnded, now: time.Now, logf: logger.Printf} move := steps.move(ctx) // the readers a rerun carried (§727, A-470d) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index 9c0cf124..bb82e620 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -553,3 +553,12 @@ func (w *world) placedBranches() map[string]map[string]string { } return out } + +// assignedOn is the modules the assignments place on node. +func (w *world) assignedOn(node string) []string { + var out []string + for _, m := range w.assignments()[node].Modules { + out = append(out, m.Module) + } + return out +}