Merge pull request 'assigner: a machine's composition across the watched branches (H-A-143, H-A-144)' (#267) from fix/47-H1-compose-across-branches into main

This commit was merged in pull request #267.
This commit is contained in:
2026-11-20 14:38:28 +00:00
8 changed files with 313 additions and 15 deletions
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strings"
"sync"
@@ -53,6 +54,13 @@ type compositionRun struct {
call func(key string, body any) (json.RawMessage, error)
// trunk is the trunk's catalog half as the assigner holds it.
trunk func() (mcc.Half, bool)
// newest is the newest catalog half the assigner's views hold of each watched branch (halves.views), and watched a module's
// watched branches in its registration's order (halves.watchedOf): a machine's composition takes each module's manifest across
// them (design 47 H-A-143, G §532). Nil: the step's revision alone.
newest func() map[string]mcc.Half
watched func(module string) []string
// assigned is the modules the assignments place on a machine (world.assignments), for the modules no watched branch lists.
assigned func(node string) []string
// running is the last `running` heard of a module on a node (world.runningOf), never the node-engine's bucket.
running func(node, module string) (ne.Running, bool)
// away says the watchman holds machine away (machine-away, not back since): the one presence the assigner keeps (world.awayNow),
@@ -123,7 +131,7 @@ func (r *compositionRun) read(ctx context.Context, module string, keys []string,
runs := r.runs.await(ctx, keys)
setPhase(ctx, meshwants.StepProgressEventPhaseReport)
for _, key := range keys {
ends = append(ends, judgeRun(module, runs[key]))
ends = append(ends, judgeRun(module, runs[key], r.unlistedOn(runs[key].Machine)))
}
out := outcomeOf(ends)
out.Shared = r.runs.sharedOf(keys, runs)
@@ -209,6 +217,9 @@ func (p *runPrep) get(ctx context.Context) error {
p.err = errors.New("the step's catalog revision is not read: " + err.Error())
return
}
if p.r.newest != nil && p.r.watched != nil {
half = acrossBranches(p.step, half, p.r.newest(), p.r.watched)
}
if p.seats, p.err = p.r.seats(ctx, half.Revision.Definitions); p.err != nil {
return
}
@@ -289,7 +300,27 @@ const notTried = "not tried: the search reached its bound"
// could not run where the module was not tried (A-470b); a machine that does not compose and blames no module fails every step on
// it, on the node's own account; a verdict with no `blamed` (a composer before §721) is judged as before, with the warning "the
// composer names no culprits"; otherwise the machine passes, with a warning for each other module blamed there.
func judgeRun(module string, run meshwants.CompositionRun) machineEnd {
//
// unlisted are the modules the machine is assigned that none of their watched branches lists: each is said on every path, in the
// machine's failed or could-not-run words where one is set, else as a warning, never dropped in silence (H-A-143, H-A-144).
func judgeRun(module string, run meshwants.CompositionRun, unlisted []string) machineEnd {
end := judgeVerdict(module, run)
for _, u := range unlisted {
said := run.Machine + " is assigned " + u + ", which no watched branch lists (H-A-143)"
switch {
case end.failed != "":
end.failed += "; " + said
case end.couldNot != "":
end.couldNot += "; " + said
default:
end.warnings = append(end.warnings, said)
}
}
return end
}
// judgeVerdict is judgeRun before the unlisted modules are said.
func judgeVerdict(module string, run meshwants.CompositionRun) machineEnd {
node := run.Machine
if run.Ended == nil {
return machineEnd{couldNot: node + ": the step stopped before its composition run " + runKeyOf(run.Candidate, node) + " ended"}
@@ -503,6 +534,74 @@ func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]string, err
return manifests, none, nil
}
// acrossBranches is the catalog half a machine's composition reads (design 47 H-A-143, H-A-144, G §532): each module's entry from
// the newest revision of the first of its watched branches, in its registration's order, that lists it, none for a module with no
// watched branches; the candidate's revision replaces its
// branch's newest for the modules it moves (its movers and the step's module), and the whole of it where it is that branch's newest
// or the branch's is not held. The half keeps the candidate's revision, its definitions with it.
func acrossBranches(step meshwants.StepRun, candidate mcc.Half, newest map[string]mcc.Half, watched func(module string) []string) mcc.Half {
halves := make(map[string]mcc.Half, len(newest)+1)
for b, h := range newest {
halves[b] = h
}
if own, held := newest[step.Branch]; !held || candidate.Revision.Revision >= own.Revision.Revision {
halves[step.Branch] = candidate
} else {
mix := mcc.Half{Revision: own.Revision, Entries: maps.Clone(own.Entries)}
moved := []string{step.Module}
for _, m := range candidate.Revision.Movers {
moved = append(moved, m.Module)
}
for _, m := range moved {
if e, ok := candidate.Entries[m]; ok {
mix.Entries[m] = e
} else {
delete(mix.Entries, m)
}
}
halves[step.Branch] = mix
}
out := mcc.Half{Revision: candidate.Revision, Entries: map[string]meshcatalog.CatalogEntry{}}
for _, h := range halves {
for name := range h.Entries {
if _, done := out.Entries[name]; done {
continue
}
for _, b := range watched(name) { // none: no watched branch lists it, it is unlisted (H-A-144)
if e, ok := halves[b].Entries[name]; ok {
out.Entries[name] = e
break
}
}
}
}
return out
}
// unlistedOn is the modules the assignments place on node that the newest revision of none of their watched branches lists, a
// module with no watched branches among them (H-A-143, H-A-144), by name.
func (r *compositionRun) unlistedOn(node string) []string {
if r.assigned == nil || r.newest == nil || r.watched == nil {
return nil
}
newest := r.newest()
var out []string
for _, m := range r.assigned(node) {
listed := false
for _, b := range r.watched(m) {
if _, ok := newest[b].Entries[m]; ok {
listed = true
break
}
}
if !listed {
out = append(out, m)
}
}
slices.Sort(out)
return slices.Compact(out)
}
// nodeEngineEntry is the entry of the module that claims the seat node-engine, or false.
func nodeEngineEntry(entries map[string]meshcatalog.CatalogEntry) (meshcatalog.CatalogEntry, bool) {
for _, name := range slices.Sorted(mapsKeys(entries)) {
@@ -616,3 +715,24 @@ func runAskPublisher(js jetstream.JetStream) func(ctx context.Context, a buildst
return err
}
}
// newestOf is the newest catalog half the views hold of each watched branch, or nil while the catalog half is not followed.
func (iv *catalogViews) newestOf() func() map[string]mcc.Half {
if iv == nil {
return nil
}
return func() map[string]mcc.Half {
if iv.h == nil {
return map[string]mcc.Half{}
}
return iv.h.views()
}
}
// watchedOf is a module's watched branches in its registration's order (halves.watchedOf); none while the half is not followed.
func (iv *catalogViews) watchedOf(module string) []string {
if iv == nil || iv.h == nil {
return nil
}
return iv.h.watchedOf(module)
}
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"maps"
"slices"
"strings"
"sync"
@@ -259,6 +260,30 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
}
}
// Another watched branch's app is in every machine's case, and a module assigned there that no watched branch lists is said in
// the machine's reason (H-A-143, §532).
across := newRunStand(t, "src2:e1", "src2:e1", "d1")
across.r.newest = func() map[string]mcc.Half {
return map[string]mcc.Half{"master": {Revision: meshcatalog.CatalogRevision{Branch: "master", Revision: 2},
Entries: map[string]meshcatalog.CatalogEntry{"f1-circuits": {Module: "f1-circuits", Fp: "f1", Manifest: map[string]any{"module": "f1-circuits"}}}}}
}
across.r.watched = func(module string) []string {
if module == "checker" || module == "lidarr" { // no registration names them; lidarr is on main's revision all the same (H-A-144)
return nil
}
return []string{"main", "master"}
}
across.r.assigned = func(node string) []string { return []string{"m", "f1-circuits", "checker", "lidarr"} }
if out := across.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed ||
!strings.Contains(out.Said, "g14 is assigned checker, which no watched branch lists (H-A-143)") ||
!strings.Contains(out.Said, "g14 is assigned lidarr, which no watched branch lists (H-A-143)") {
t.Errorf("the step across the watched branches came to %+v", out)
}
if c := across.requests["ace"].Cases[0]; string(c.Manifests["f1-circuits"].Raw) != `{"module":"f1-circuits"}` || len(c.Manifests) != 4 ||
c.Manifests["lidarr"].Raw != nil {
t.Errorf("the case across the watched branches holds %v", slices.Sorted(maps.Keys(c.Manifests)))
}
// At the inputs state's cap the ask is refused, naming the store (§354).
full := newRunStand(t, "src2:e1", "src2:e1", "d1")
full.r.keeper.used = func() (int64, error) { return inputsCap, nil }
@@ -438,21 +463,33 @@ func TestAMachineIsJudgedByTheModulesItBlames(t *testing.T) {
run meshwants.CompositionRun
failed, couldNo string
warnings []string
unlisted []string
}{
{"blamed module fails", "a", runOf(blamesA), "ace holds a: /etc/x is a's and c's", "", nil},
{"another module passes with a warning", "b", runOf(blamesA), "", "", []string{"ace holds a: /etc/x is a's and c's"}},
{"blames no module fails both", "b", runOf(`{"composes":false,"reason":"the node's own composition","blamed":[]}`), "ace does not compose and blames no module: the node's own composition", "", nil},
{"an old composer", "b", runOf(`{"composes":false,"reason":"x"}`), "ace does not compose: x", "", []string{"ace: the composer names no culprits"}},
{"composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "", nil},
{"blamed module fails", "a", runOf(blamesA), "ace holds a: /etc/x is a's and c's", "", nil, nil},
{"another module passes with a warning", "b", runOf(blamesA), "", "", []string{"ace holds a: /etc/x is a's and c's"}, nil},
{"blames no module fails both", "b", runOf(`{"composes":false,"reason":"the node's own composition","blamed":[]}`), "ace does not compose and blames no module: the node's own composition", "", nil, nil},
{"an old composer", "b", runOf(`{"composes":false,"reason":"x"}`), "ace does not compose: x", "", []string{"ace: the composer names no culprits"}, nil},
{"composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "", nil, nil},
{"own module not tried", "a", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "",
"ace holds a untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", nil},
"ace holds a untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", nil, nil},
{"another not tried", "b", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "", "",
[]string{"ace holds a: not tried: the search reached its bound"}},
{"not judged", "a", runOf(`{"composes":false,"not_judged":"no seat list","blamed":[]}`), "", "ace was not judged: no seat list", nil},
{"could not run", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"}, "", "ace: gone", nil},
[]string{"ace holds a: not tried: the search reached its bound"}, nil},
{"not judged", "a", runOf(`{"composes":false,"not_judged":"no seat list","blamed":[]}`), "", "ace was not judged: no seat list", nil, nil},
{"could not run", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"}, "", "ace: gone", nil, nil},
// H-A-143: a module assigned there that no watched branch lists is said in the reason, or as a warning where it composes.
{"an unlisted module in the reason", "b", runOf(`{"composes":false,"reason":"nothing provides \"photos\"","blamed":[]}`),
"ace does not compose and blames no module: nothing provides \"photos\"; ace is assigned photos, which no watched branch lists (H-A-143)", "", nil, []string{"photos"}},
{"does not compose, blames a module, an unlisted module", "a", runOf(`{"composes":false,"reason":"nothing provides \"photos\"","blamed":[{"module":"a","problem":"nothing provides \"photos\""}]}`),
"ace holds a: nothing provides \"photos\"; ace is assigned photos, which no watched branch lists (H-A-143)", "", nil, []string{"photos"}},
{"another module blamed, an unlisted module", "b", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"x"}]}`), "", "",
[]string{"ace holds a: x", "ace is assigned photos, which no watched branch lists (H-A-143)"}, []string{"photos"}},
{"could not run, an unlisted module", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"},
"", "ace: gone; ace is assigned photos, which no watched branch lists (H-A-143)", nil, []string{"photos"}},
{"an unlisted module where it composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "",
[]string{"ace is assigned photos, which no watched branch lists (H-A-143)"}, []string{"photos"}},
} {
t.Run(tc.name, func(t *testing.T) {
e := judgeRun(tc.module, tc.run)
e := judgeRun(tc.module, tc.run, tc.unlisted)
if e.failed != tc.failed || e.couldNot != tc.couldNo || !slices.Equal(e.warnings, tc.warnings) {
t.Errorf("judged %+v, want failed %q, could not run %q, warnings %q", e, tc.failed, tc.couldNo, tc.warnings)
}
@@ -572,3 +609,67 @@ func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) {
}
}
}
// Design 47 H-A-143, G §532: a machine's composition takes each module's entry from the newest revision of the first of its
// watched branches, in its registration's order, that lists it; the candidate's revision replaces its branch's newest for the
// modules it moves, and whole where it is that branch's newest; the half keeps the candidate's revision.
func TestACompositionTakesEachModuleAcrossTheWatchedBranches(t *testing.T) {
entry := func(module, fp string) meshcatalog.CatalogEntry {
return meshcatalog.CatalogEntry{Module: module, Fp: fp, Manifest: map[string]any{"module": module}}
}
half := func(branch string, revision int64, movers []string, entries ...meshcatalog.CatalogEntry) mcc.Half {
h := mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: branch, Revision: revision, Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: branch}}},
Entries: map[string]meshcatalog.CatalogEntry{}}
for _, m := range movers {
h.Revision.Movers = append(h.Revision.Movers, meshcatalog.TakeInEntry{Module: m})
}
for _, e := range entries {
h.Entries[e.Module] = e
}
return h
}
newest := map[string]mcc.Half{
"main": half("main", 4, nil, entry("engine", "e1"), entry("a", "a1"), entry("both", "main")),
"master": half("master", 9, nil, entry("f1-circuits", "f1")),
"nox-mesh": half("nox-mesh", 3, nil, entry("link2pay", "l1"), entry("photos", "p1"), entry("both", "nox")),
}
registrations := map[string][]string{"both": {"nox-mesh", "main"}, "f1-circuits": {"master"}, "link2pay": {"nox-mesh"}, "photos": {"nox-mesh"}}
registrations["engine"], registrations["a"] = []string{"main"}, []string{"main"}
watched := func(module string) []string { return registrations[module] } // a module no registration names has none (H-A-144)
for _, tc := range []struct {
name string
step meshwants.StepRun
candidate mcc.Half
newest map[string]mcc.Half
want map[string]string // module -> fingerprint
}{
{"the newest of main, the others' apps beside it", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5},
half("main", 5, []string{"a"}, entry("engine", "e1"), entry("a", "a2"), entry("both", "main")), newest,
map[string]string{"engine": "e1", "a": "a2", "both": "nox", "f1-circuits": "f1", "link2pay": "l1", "photos": "p1"}},
{"an older candidate moves only its movers", meshwants.StepRun{Branch: "nox-mesh", Module: "link2pay", Revision: 2},
half("nox-mesh", 2, []string{"photos"}, entry("link2pay", "l0"), entry("both", "nox0")), newest,
map[string]string{"engine": "e1", "a": "a1", "both": "nox", "f1-circuits": "f1", "link2pay": "l0"}},
{"a branch not yet held is the candidate's", meshwants.StepRun{Branch: "master", Module: "f1-circuits", Revision: 1},
half("master", 1, nil, entry("f1-circuits", "f0")), map[string]mcc.Half{"main": newest["main"]},
map[string]string{"engine": "e1", "a": "a1", "both": "main", "f1-circuits": "f0"}},
{"no view held yet: the candidate alone", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5},
half("main", 5, nil, entry("a", "a2")), nil, map[string]string{"a": "a2"}},
{"a module no registration names is not composed", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5},
half("main", 5, nil, entry("a", "a2"), entry("checker", "c1")), nil, map[string]string{"a": "a2"}},
} {
t.Run(tc.name, func(t *testing.T) {
got := acrossBranches(tc.step, tc.candidate, tc.newest, watched)
fps := map[string]string{}
for name, e := range got.Entries {
fps[name] = e.Fp
}
if !maps.Equal(fps, tc.want) {
t.Errorf("composed from %v, want %v", fps, tc.want)
}
if got.Revision.Revision != tc.candidate.Revision.Revision || len(got.Revision.Definitions) != 1 || got.Revision.Definitions[0].Digest != tc.step.Branch {
t.Errorf("the half keeps revision %d with definitions %v, want the candidate's %d with its own", got.Revision.Revision, got.Revision.Definitions,
tc.candidate.Revision.Revision)
}
})
}
}
@@ -818,7 +818,7 @@ func (r *compositionRuns) leave(pipeline string, stop func(ids []string)) ([]str
// passedOn is whether the step of module passed on the machine of an ended run (§722): neither failed nor could not run there.
func passedOn(module string, run meshwants.CompositionRun) bool {
e := judgeRun(module, run)
e := judgeRun(module, run, nil) // the unlisted modules change only the words, never whether it passed
return e.failed == "" && e.couldNot == ""
}
@@ -416,7 +416,7 @@ func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) {
len(*drops) != 1 || !strings.HasPrefix((*drops)[0], "run-1: ") {
t.Fatalf("past the bound: %+v, drops %v", run, *drops)
}
if got := judgeRun("a", run); got.couldNot != run.Said {
if got := judgeRun("a", run, nil); got.couldNot != run.Said {
t.Errorf("a reader of the run past its bound: %+v", got)
}
late := buildstep.StepEnded{Ask: "run-1", Asker: runAsker, Outcome: buildstep.OutcomePassed, Output: "{}"}
@@ -6,6 +6,7 @@ import (
"encoding/hex"
"encoding/json"
"slices"
"strings"
"sync"
"time"
@@ -13,6 +14,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/views"
)
@@ -191,3 +193,25 @@ func (h *halves) views() map[string]mcc.Half {
}
return out
}
// watchedOf is module's watched branches in its registration's order (the branch `register` named first, then each branch in the
// order `watch` added it). Once registrations are held, a module none names has none: it is unlisted (H-A-144). Only before the
// first registrations are held do the gate's source's branches stand in (the branches followed then), in the order the source
// gives them, by their names escaped as its keys are (design 47 G §532, H-A-143, H-A-144).
func (h *halves) watchedOf(module string) []string {
h.mu.Lock()
b := h.branches
h.mu.Unlock()
if b == nil {
return nil
}
if b.Registrations != nil && len(b.Registrations.All()) > 0 {
if reg, ok := b.Registrations.Registration(module); ok {
return slices.Clone(reg.Branches)
}
return nil
}
out := b.Followed()
slices.SortStableFunc(out, func(x, y string) int { return strings.Compare(keys.EscapeName(x), keys.EscapeName(y)) })
return out
}
@@ -3,6 +3,8 @@ package main
import (
"context"
"encoding/json"
"errors"
"slices"
"testing"
"time"
@@ -48,3 +50,45 @@ func TestAMeshThatNamesNoBranchSettlesTheAssignersFirstPass(t *testing.T) {
}
}
}
// Design 47 H-A-143, H-A-144, G §532: a module's watched branches are its registration's, in its order; once registrations are held a
// module none names has none; before the first registrations the gate's source's branches stand in, in the source's order.
func TestAModulesWatchedBranchesAreItsRegistrations(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
for _, tc := range []struct {
name string
regs []seat.Registration
module string
want []string
}{
{"its registration's order", []seat.Registration{{Module: "link2pay", Branches: []string{"nox-mesh", "main"}}}, "link2pay", []string{"nox-mesh", "main"}},
{"no registration names it", []seat.Registration{{Module: "link2pay", Branches: []string{"nox-mesh", "main"}}}, "checker", nil},
{"before the first registrations, the source's", []seat.Registration{}, "checker", []string{"main", "master"}},
} {
t.Run(tc.name, func(t *testing.T) {
call := func(key string, _ any) (json.RawMessage, error) {
switch key {
case mcc.RegistrationsVerb:
return json.Marshal(seat.RegistrationsAnswer{Registrations: tc.regs})
case mcc.SourceVerb:
return json.Marshal(map[string][]string{"branches": {"master", "main"}})
}
return nil, errors.New("not answered here")
}
reader := &views.ProcessReader{Module: "mesh-delivery-assigner", Call: call, Logf: t.Logf}
h := &halves{w: newWorld(), call: call, reader: reader, wake: func() {}, logf: t.Logf, ctx: ctx}
if err := h.follow(); err != nil {
t.Fatal(err)
}
for _, f := range reader.Followers()[:2] { // the registrations and the gate's source
if err := f.Start(ctx); err != nil {
t.Fatal(err)
}
}
if got := h.watchedOf(tc.module); !slices.Equal(got, tc.want) {
t.Errorf("the watched branches of %s are %v, want %v", tc.module, got, tc.want)
}
})
}
}
@@ -665,7 +665,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
// The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine.
if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil {
run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept,
trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf}
trunk: trunkOf(iv, w.trunkName), newest: iv.newestOf(), watched: iv.watchedOf, assigned: w.assignedOn, running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf}
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume,
validate: run.validate, resumeValidate: run.resumeValidate, ended: runs.stepEnded, now: time.Now, logf: logger.Printf}
move := steps.move(ctx) // the readers a rerun carried (§727, A-470d)
@@ -553,3 +553,12 @@ func (w *world) placedBranches() map[string]map[string]string {
}
return out
}
// assignedOn is the modules the assignments place on node.
func (w *world) assignedOn(node string) []string {
var out []string
for _, m := range w.assignments()[node].Modules {
out = append(out, m.Module)
}
return out
}