From ba11ff58dd91efdabdc4ffa3ed7db71b51cd5202 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 11:36:15 +0100 Subject: [PATCH] assigner: a composition run's bound times owed progress, its untaken ask its own (A4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A run whose build node stops saying progress, or whose ask no build node takes, would hold its readers for ever: ask-taken arms bound. and each step-progress arms it again; untaken. times the queue from the start, the readers said waiting for a build node meanwhile; an expiry drops the ask and ends the run could not run naming the bound, and a step-ended heard after it is ignored and journaled. Readers of a run left open at a restart are taken up from the keys. Builds: design 47 G §719 (take-up at start), §720 (the bound, when the clock ends, a run never taken, what the assigner hears); A-466q, A-466s, A-468, A-468b, A-469b, A-469s, A-469y, A-470 --- .../compositionbounds.go | 263 ++++++++++++++++++ .../mesh-delivery-assigner/compositionrun.go | 20 +- .../mesh-delivery-assigner/compositionruns.go | 22 +- .../compositionruns_test.go | 130 ++++++++- .../mesh-delivery-assigner/compositionstep.go | 17 +- .../cmd/mesh-delivery-assigner/main.go | 18 +- .../mesh-delivery-assigner/manifest_test.go | 5 +- modules/mesh-delivery-assigner/module.json | 3 + 8 files changed, 460 insertions(+), 18 deletions(-) create mode 100644 modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go new file mode 100644 index 00000000..f1964a2c --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go @@ -0,0 +1,263 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go/jetstream" + + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// A composition run's bound (design 47 G §720; A-466n, A-466p, A-466q, A-466s, A-468b, A-469b, A-469s, A-469y, A-470): a run +// has one clock, the key `bound.` of composition-runs with its own time to live composition-step-bound, written at the +// build seat's ask-taken of the run's ask and written again at each step-progress of that ask; an ask still waiting in the queue +// has no clock but `untaken.`, written at the run's start with the time to live run-untaken-bound. Either's expiry, +// heard as the bus's event, ends the run: its ask dropped through the build seat's drop, the run ended could not run naming the +// ask and the bound, so every reader ends could not run with it. While the ask waits for a build node each reader is said waiting +// by a wait-held on mesh-wants (`on: build node`), ended `ran` at ask-taken or `superseded` when the run ends untaken. + +// waitOnBuildNode is what a composition run's waiting reader waits on (A-469b, A-469y). +const waitOnBuildNode = "build node" + +// dropVerbKey is the Go build seat's drop, asked as a kept call (§119, A-182). +const dropVerbKey = "seat:" + nodebuildgo.Seat + "." + nodebuildgo.VerbDrop + +// dropOver asks the Go build seat's drop of a run ask as a kept call, of any holder, which records it (A-469s). +func dropOver(call func(key string, body any) (json.RawMessage, error)) func(ask, why string) error { + return func(ask, why string) error { + raw, err := call(dropVerbKey, nodebuildgo.DropInput{ID: ask, Why: why, By: runAsker}) + if err != nil { + return err + } + if _, said, ok := buscalls.Unwrap(raw); !ok { + return fmt.Errorf("%s answered: %s", dropVerbKey, said) + } + return nil + } +} + +// waitOf is a reader's wait for a build node to take its run's ask (A-469y). +func waitOf(run meshwants.CompositionRun, reader meshwants.CompositionReader) meshwants.Wait { + w := meshwants.Wait{Step: "pipeline", Owner: meshwants.Seat, Module: reader.Module, Node: run.Machine, On: waitOnBuildNode, Fp: reader.Fingerprint, + Words: fmt.Sprintf("the composition step of %s waits for a build node to take the composition run %s of %s (ask %s)", reader.Pipeline, + runKeyOf(run.Candidate, run.Machine), run.Machine, run.Ask)} + if run.Started != nil { + w.Since = *run.Started + } + return w +} + +// sayWaits says, for each reader, its wait held or ended (how: ran, superseded); a wait that cannot be said is journaled. +func (r *compositionRuns) sayWaits(run meshwants.CompositionRun, readers []meshwants.CompositionReader, how string) { + if r.say == nil { + return + } + for _, c := range readers { + w := waitOf(run, c) + event, id := meshwants.EventWaitHeld, "composition."+runKeyOf(run.Candidate, run.Machine)+"."+run.Ask+"."+c.ID+".held" + if how != "" { + w.How, event, id = how, meshwants.EventWaitEnded, strings.TrimSuffix(id, ".held")+".ended" + } + if err := r.say(event, w, id); err != nil { + r.logf("the %s of %s on the composition run of %s is not said: %v", event, c.Pipeline, run.Machine, err) + } + } +} + +// untaken is whether the run's ask has not been taken yet: its untaken key stands. +func (r *compositionRuns) untaken(key string) bool { + raw, _, err := r.kv.get(runUntakenPrefix + key) + return err == nil && raw != nil +} + +// heardAskTaken begins a run's clock at its ask's ask-taken: the bound key armed, the untaken key deleted, each reader's wait +// ended ran. An ask no running run names is another asker's. +func (r *compositionRuns) heardAskTaken(e nodebuildgo.AskTaken) error { + key, run, err := r.runOfAsk(e.Asker, e.Ask) + if err != nil || key == "" { + return err + } + if err := r.kv.arm(runBoundPrefix+key, r.set.stepBound); err != nil { + return fmt.Errorf("the bound of the composition run %s is not armed: %w", key, err) + } + if r.untaken(key) { + if err := r.kv.del(runUntakenPrefix + key); err != nil { + return err + } + r.sayWaits(run, run.Readers, "ran") + } + return nil +} + +// heardProgress begins a running run's clock again at each step-progress of its ask (§720: owed progress, not run length). +func (r *compositionRuns) heardProgress(e nodebuildgo.StepProgress) error { + key, _, err := r.runOfAsk(e.Asker, e.Ask) + if err != nil || key == "" { + return err + } + return r.kv.arm(runBoundPrefix+key, r.set.stepBound) +} + +// runOfAsk is the running run of an ask the assigner put, or none. +func (r *compositionRuns) runOfAsk(asker, ask string) (string, meshwants.CompositionRun, error) { + if asker != runAsker || ask == "" { + return "", meshwants.CompositionRun{}, nil + } + key, run, err := r.keyOfAsk(ask) + if err != nil || key == "" || run.Ended != nil { + return "", meshwants.CompositionRun{}, err + } + return key, run, nil +} + +// expired ends the run whose bound key's time to live ended (§720): its ask dropped, the run ended could not run naming the +// ask and the bound. A run that ended already, or a key of none, takes nothing. +func (r *compositionRuns) expired(boundKey string) { + key, bound, setting := "", time.Duration(0), "" + switch { + case strings.HasPrefix(boundKey, runBoundPrefix): + key, bound, setting = strings.TrimPrefix(boundKey, runBoundPrefix), r.set.stepBound, "composition-step-bound" + case strings.HasPrefix(boundKey, runUntakenPrefix): + key, bound, setting = strings.TrimPrefix(boundKey, runUntakenPrefix), r.set.untaken, "run-untaken-bound" + default: + return + } + run, _, ok, err := r.read(key) + if err != nil || !ok || run.Ended != nil { + return + } + said := fmt.Sprintf("%s: its run ask %s said no progress within %s (%s)", run.Machine, run.Ask, bound, setting) + if setting == "run-untaken-bound" { + said = fmt.Sprintf("%s: its run ask %s was taken by no build node within %s (%s)", run.Machine, run.Ask, bound, setting) + } + if r.drop != nil { + if err := r.drop(run.Ask, said); err != nil { + r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err) + } + } + if _, err := r.endRun(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil); err != nil { + r.logf("the composition run %s past its bound is not ended: %v", key, err) + } +} + +// endRun ends a run (end) and clears its bounds: the untaken key's readers said superseded, both bound keys deleted. +func (r *compositionRuns) endRun(key, ask, outcome, said string, verdict map[string]any) (bool, error) { + untaken := r.untaken(key) + ended, err := r.end(key, ask, outcome, said, verdict) + if err != nil || !ended { + return ended, err + } + if untaken { + if run, _, ok, err := r.read(key); err == nil && ok { + r.sayWaits(run, run.Readers, "superseded") + } + } + for _, k := range []string{runBoundPrefix + key, runUntakenPrefix + key} { + if err := r.kv.del(k); err != nil { + r.logf("the bound key %s of the ended composition run is not deleted: %v", k, err) + } + } + return true, nil +} + +// readersOfOpen are the readers of every run with no `ended`, each with its ask's id once: what the assigner takes up again at +// its start (§719), each reader's end said from the keys alone (A-468). +func (r *compositionRuns) readersOfOpen() ([]meshwants.CompositionReader, error) { + all, err := r.kv.keys() + if err != nil { + return nil, err + } + seen := map[string]bool{} + var out []meshwants.CompositionReader + for _, key := range all { + if !isRunKey(key) { + continue + } + run, _, ok, err := r.read(key) + if err != nil || !ok || run.Ended != nil { + continue + } + for _, c := range run.Readers { + if !seen[c.ID] { + seen[c.ID] = true + out = append(out, c) + } + } + } + return out, nil +} + +// keysOf are the keys of the candidate's runs that the reader reads. +func (r *compositionRuns) keysOf(candidate, id string) ([]string, error) { + all, err := r.kv.keys() + if err != nil { + return nil, err + } + var out []string + prefix := keysPart(candidate) + "." + for _, key := range all { + if !isRunKey(key) || !strings.HasPrefix(key, prefix) { + continue + } + if run, _, ok, err := r.read(key); err == nil && ok && hasReader(run.Readers, id) { + out = append(out, key) + } + } + return out, nil +} + +// hearRunBounds hears the build seats' ask-taken and step-progress on the assigner's durable consumer (§720, A-466q): each +// handed to the runs; one that cannot be written is given again by the bus. +func hearRunBounds(hub interface { + Handle(string, func(jetstream.Msg)) error +}, r *compositionRuns, logf func(string, ...any)) error { + if err := hub.Handle(nodebuildgo.SubjectAskTakenAll, func(m jetstream.Msg) { + var e nodebuildgo.AskTaken + if err := json.Unmarshal(m.Data(), &e); err != nil { + logf("%s holds what is no ask-taken, left: %v", m.Subject(), err) + _ = m.Ack() + return + } + ackOrNak(m, r.heardAskTaken(e), logf) + }); err != nil { + return err + } + return hub.Handle(nodebuildgo.SubjectStepProgressAll, func(m jetstream.Msg) { + var e nodebuildgo.StepProgress + if err := json.Unmarshal(m.Data(), &e); err != nil { + logf("%s holds what is no step-progress, left: %v", m.Subject(), err) + _ = m.Ack() + return + } + ackOrNak(m, r.heardProgress(e), logf) + }) +} + +func ackOrNak(m jetstream.Msg, err error, logf func(string, ...any)) { + if err != nil { + logf("%s is not taken, so the bus gives it again: %v", m.Subject(), err) + _ = m.Nak() + return + } + _ = m.Ack() +} + +// takeUp says, at the assigner's start, the end of each reader of a run with no `ended` once its runs end (§719, A-468): the +// step's goroutine is gone, so its end is judged from the candidate's keys naming it. +func (c *compositionSteps) takeUp(ctx context.Context, r *compositionRuns) { + readers, err := r.readersOfOpen() + if err != nil { + c.logf("composition-runs: the runs standing at the start are not read, so their readers are not taken up: %v", err) + return + } + for _, reader := range readers { + c.logf("the composition step %s of %s is taken up again by its runs (ask %s)", reader.Pipeline, reader.Module, reader.ID) + go c.say1(ctx, reader.ID, stepRunOf(reader), c.resume) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go index fff06474..788e1758 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go @@ -145,6 +145,24 @@ func (r *compositionRun) run(ctx context.Context, id string, step meshwants.Step return outcomeOf(ends) } +// resume judges a step taken up at the assigner's start (§719, A-468): the runs of its candidate naming it, once each has ended. +func (r *compositionRun) resume(ctx context.Context, id string, step meshwants.StepRun) stepOutcome { + candidate, err := candidateKeyOf(step) + if err != nil { + return couldNotRun(err.Error()) + } + keys, err := r.runs.keysOf(candidate, id) + if err != nil { + return couldNotRun("the step's composition runs are not read: " + err.Error()) + } + runs := r.runs.await(ctx, keys) + var ends []machineEnd + for _, key := range keys { + ends = append(ends, judgeRun(step.Module, runs[key])) + } + return outcomeOf(ends) +} + // outcomeOf is a step's outcome from its machines: failed before could not run before passed (§354); a step that judged no // machine never passes (A-467a), each machine not judged named with why. func outcomeOf(ends []machineEnd) stepOutcome { @@ -226,7 +244,7 @@ func (p *runPrep) get(ctx context.Context) error { func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep) { node, ask := run.Machine, run.Ask end := func(said string) { - if _, err := r.runs.end(key, ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil); err != nil { + if _, err := r.runs.endRun(key, ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil); err != nil { r.logf("the composition run %s is not ended: %v", key, err) } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go index f097eace..88208bfa 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go @@ -40,9 +40,12 @@ const ( // runKeyOf is the key of a candidate's composition run of a machine (§719). func runKeyOf(candidate, machine string) string { - return keys.EscapeName(candidate) + "." + keys.EscapeName(machine) + return keysPart(candidate) + "." + keys.EscapeName(machine) } +// keysPart is a name as a key's part. +func keysPart(name string) string { return keys.EscapeName(name) } + // isRunKey is whether a key of the state is a run's own, not one of its bound keys. func isRunKey(key string) bool { return !strings.HasPrefix(key, runBoundPrefix) && !strings.HasPrefix(key, runUntakenPrefix) @@ -215,6 +218,10 @@ type compositionRuns struct { mu sync.Mutex // changed is closed and replaced at each change of the state heard (heard): a step waiting on a run reads its key again. changed chan struct{} + // say says an event of the seat mesh-wants (a reader's wait, A-469y); drop asks the build seat's drop of a run ask (§119); + // nil says or drops nothing. + say publisher + drop func(ask, why string) error } func newCompositionRuns(kv runsKV, set runSettings, now func() time.Time, logf func(string, ...any)) *compositionRuns { @@ -315,6 +322,9 @@ func (r *compositionRuns) join(reader meshwants.CompositionReader, candidate, ma return joined{}, err } r.heard() + if run.Ended == nil && r.untaken(key) { + r.sayWaits(run, []meshwants.CompositionReader{reader}, "") + } return joined{key: key, run: run}, nil } if err := r.room(); err != nil { @@ -338,6 +348,11 @@ func (r *compositionRuns) join(reader meshwants.CompositionReader, candidate, ma return joined{}, err } r.heard() + // An ask waiting in the queue has no clock but run-untaken-bound from its start (A-466s, A-468b, A-470). + if err := r.kv.arm(runUntakenPrefix+key, r.set.untaken); err != nil { + r.logf("the untaken bound of the composition run %s is not armed: %v", key, err) + } + r.sayWaits(fresh, fresh.Readers, "") return joined{key: key, started: true, run: fresh}, nil } } @@ -488,7 +503,8 @@ func (r *compositionRuns) heardEnd(e buildstep.StepEnded, read func(buildstep.St if err != nil || key == "" { return err } - if run.Ended != nil { + if run.Ended != nil { // ended by its bound or a drop: the end heard later is ignored (§720) + r.logf("the step-ended of the run ask %s is ignored: its composition run %s ended before it (%s)", e.Ask, key, run.Said) return nil } node := run.Machine @@ -501,7 +517,7 @@ func (r *compositionRuns) heardEnd(e buildstep.StepEnded, read func(buildstep.St } else if verdict, said = runVerdict(node, answer); said == "" { outcome = meshwants.CompositionRunOutcomePassed } - _, err = r.end(key, e.Ask, outcome, said, verdict) + _, err = r.endRun(key, e.Ask, outcome, said, verdict) return err } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go index 03171fd6..a7f73491 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "fmt" "os" "path/filepath" "slices" @@ -11,6 +12,8 @@ import ( "time" meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" ) // runsMem is the state composition-runs in memory, its bound keys' times to live kept beside. @@ -129,10 +132,14 @@ func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) { }() } wg.Wait() + runKeys := func() []string { + all, _ := kv.keys() + return slices.DeleteFunc(all, func(k string) bool { return !isRunKey(k) }) + } if got[0].started == got[1].started { t.Fatalf("started %v and %v: want exactly one step to start the run", got[0].started, got[1].started) } - keys, _ := kv.keys() + keys := runKeys() run, _, _, _ := r.read(runKeyOf("main.3", "n1")) if len(keys) != 1 || len(run.Readers) != 2 || run.Candidate != "main.3" || run.Machine != "n1" || run.Started == nil { t.Errorf("keys %v, run %+v: want one key read by both", keys, run) @@ -194,6 +201,12 @@ func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) { t.Fatal(err) } keys, _ := kv.keys() + for _, k := range []string{runBoundPrefix, runUntakenPrefix} { + if v, _, _ := kv.get(k + runKeyOf("main.1", "n1")); v != nil { + t.Errorf("the key %s stays past its run", k+runKeyOf("main.1", "n1")) + } + } + keys = slices.DeleteFunc(keys, func(k string) bool { return !isRunKey(k) }) if want := []string{runKeyOf("main.2", "n1"), runKeyOf("main.9", "n1")}; !slices.Equal(keys, want) { t.Errorf("keys %v, want %v: a run ended more than a day ago goes with its bound key at the next run", keys, want) } @@ -227,3 +240,118 @@ func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) { t.Errorf("a reader names no ask id: %s", raw) } } + +// Design 47 G §720, A-466s, A-468b, A-469y, A-470: a run's untaken bound is armed at its start and each reader said waiting for a +// build node; ask-taken arms the bound of composition-step-bound and ends the waits ran; each step-progress arms it again; the +// bound's expiry drops the ask and ends the run could not run naming the ask and the bound, a step-ended heard later ignored and +// journaled; the untaken bound's expiry ends it naming run-untaken-bound, the waits superseded; the readers of a run with no end +// are taken up at the start. +func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) { + now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC) + reader := func(id string) meshwants.CompositionReader { + return readerOf(id, meshwants.StepRun{Pipeline: "main.3." + id, Step: meshwants.StepRunStepComposition, Module: id, Branch: "main", Revision: 3}) + } + type said struct{ event, how, module string } + setup := func(t *testing.T) (*runsMem, *compositionRuns, *[]said, *[]string, *[]string) { + kv := newRunsMem() + var saids []said + var drops, journal []string + var mu sync.Mutex + r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, func(f string, a ...any) { + mu.Lock() + journal = append(journal, fmt.Sprintf(f, a...)) + mu.Unlock() + }) + r.say = func(event string, body any, _ string) error { + w := body.(meshwants.Wait) + if w.On != waitOnBuildNode || w.Step != "pipeline" || w.Owner != meshwants.Seat || w.Node == "" { + t.Errorf("the wait said %+v", w) + } + saids = append(saids, said{event, w.How, w.Module}) + return nil + } + r.drop = func(ask, why string) error { drops = append(drops, ask+": "+why); return nil } + return kv, r, &saids, &drops, &journal + } + key := runKeyOf("main.3", "n1") + t.Run("taken, progress, bound", func(t *testing.T) { + kv, r, saids, drops, journal := setup(t) + if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil { + t.Fatal(err) + } + if _, err := r.join(reader("b"), "main.3", "n1", "run-2"); err != nil { + t.Fatal(err) + } + if kv.ttl[runUntakenPrefix+key] != 24*time.Hour || !slices.Equal(*saids, []said{{meshwants.EventWaitHeld, "", "a"}, {meshwants.EventWaitHeld, "", "b"}}) { + t.Fatalf("at the start: untaken %v, said %v", kv.ttl, *saids) + } + if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: "someone-else"}); err != nil || kv.ttl[runBoundPrefix+key] != 0 { + t.Fatalf("another asker's ask-taken arms a bound: %v %v", kv.ttl, err) + } + if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: runAsker, Node: "b1"}); err != nil { + t.Fatal(err) + } + _, untaken, _ := kv.get(runUntakenPrefix + key) + if kv.ttl[runBoundPrefix+key] != 30*time.Minute || untaken != 0 || len(*saids) != 4 || (*saids)[2].how != "ran" || (*saids)[3].how != "ran" { + t.Fatalf("at ask-taken: %v, said %v", kv.ttl, *saids) + } + before := kv.revs[runBoundPrefix+key] + if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil || kv.revs[runBoundPrefix+key] == before { + t.Fatalf("a step-progress does not arm the bound again: %v", err) + } + r.expired(runBoundPrefix + key) + run, _, _, _ := r.read(key) + if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 said no progress within 30m0s (composition-step-bound)" || + len(*drops) != 1 || !strings.HasPrefix((*drops)[0], "run-1: ") { + t.Fatalf("past the bound: %+v, drops %v", run, *drops) + } + if got := judgeRun("a", run); got.couldNot != run.Said { + t.Errorf("a reader of the run past its bound: %+v", got) + } + late := buildstep.StepEnded{Ask: "run-1", Asker: runAsker, Outcome: buildstep.OutcomePassed, Output: "{}"} + if err := r.heardEnd(late, func(e buildstep.StepEnded) ([]byte, error) { return []byte(e.Output), nil }); err != nil { + t.Fatal(err) + } + if again, _, _, _ := r.read(key); again.Said != run.Said || !slices.ContainsFunc(*journal, func(l string) bool { + return strings.Contains(l, "step-ended of the run ask run-1 is ignored") && strings.Contains(l, key) + }) { + t.Errorf("a step-ended after the bound: %+v, journal %v", again, *journal) + } + if keys, _ := kv.keys(); !slices.Equal(keys, []string{key}) { + t.Errorf("the bound keys stay past the run's end: %v", keys) + } + }) + t.Run("untaken", func(t *testing.T) { + _, r, saids, drops, _ := setup(t) + if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil { + t.Fatal(err) + } + r.expired(runUntakenPrefix + key) + run, _, _, _ := r.read(key) + if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 was taken by no build node within 24h0m0s (run-untaken-bound)" || + len(*drops) != 1 || !slices.Equal(*saids, []said{{meshwants.EventWaitHeld, "", "a"}, {meshwants.EventWaitEnded, "superseded", "a"}}) { + t.Errorf("untaken past its bound: %+v, drops %v, said %v", run, *drops, *saids) + } + r.expired(runUntakenPrefix + key) // a run ended takes nothing + if len(*drops) != 1 { + t.Errorf("an ended run is dropped again: %v", *drops) + } + }) + t.Run("taken up at the start", func(t *testing.T) { + _, r, _, _, _ := setup(t) + for _, c := range []struct{ id, machine string }{{"a", "n1"}, {"a", "n2"}, {"b", "n1"}} { + if _, err := r.join(reader(c.id), "main.3", c.machine, "run-"+c.machine); err != nil { + t.Fatal(err) + } + } + ended := now + _, _, _ = r.change(runKeyOf("main.3", "n2"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &ended, "passed"; return true }) + readers, err := r.readersOfOpen() + if err != nil || len(readers) != 2 || readers[0].ID != "a" || readers[1].ID != "b" { + t.Fatalf("the readers taken up: %+v %v", readers, err) + } + if keys, _ := r.keysOf("main.3", "a"); !slices.Equal(keys, []string{runKeyOf("main.3", "n1"), runKeyOf("main.3", "n2")}) { + t.Errorf("the runs of reader a: %v", keys) + } + }) +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go index 48cedf7e..899be142 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go @@ -30,10 +30,11 @@ type compositionSteps struct { node string // say says one event of the seat mesh-wants (wantsPublisher). say publisher - // run runs one step and answers what it came to. - run func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome - now func() time.Time - logf func(string, ...any) + // run runs one step and answers what it came to; resume judges a step taken up at the start from its runs (takeUp). + run func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome + resume func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome + now func() time.Time + logf func(string, ...any) // asker is the name the step's progress names as its asker, read when it is said (stepAskerOf). asker func() string // ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own. @@ -100,16 +101,16 @@ func (c *compositionSteps) take(ctx context.Context, id string, data []byte) boo c.logf("a step ask that is no composition step with an id is left: %v", err) return false } - go c.runOne(ctx, id, run) + go c.say1(ctx, id, run, c.run) return true } -// runOne runs one step and says its end. -func (c *compositionSteps) runOne(ctx context.Context, id string, run meshwants.StepRun) { +// say1 runs one step with how and says its end. +func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome) { started := c.now().UTC() phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch} stop := c.sayProgress(id, phase) - o := c.run(context.WithValue(ctx, phaseKey{}, phase), id, run) + o := how(context.WithValue(ctx, phaseKey{}, phase), id, run) stop() ended := c.now().UTC() end := meshwants.StepEnded{V: 1, ID: id, Pipeline: run.Pipeline, Step: meshwants.StepEndedStepComposition, Module: run.Module, diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 3dabcb87..337c5bba 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -538,8 +538,15 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, return err } runs = newCompositionRuns(busRuns{ctx, kv}, set, time.Now, logger.Printf) - if err := watchRuns(ctx, kv, runs, nil, nil); err != nil { - return fmt.Errorf("the state %s is not watched: %w", buckets.CompositionRunsState, err) + if js != nil { + runs.say = wantsPublisher(ctx, js, thisNode) + } + if bus != nil { + runs.drop = dropOver(bus.CallKept) + } + // The build seats' ask-taken and step-progress begin a run's clock (§720, A-466q). + if err := hearRunBounds(hub, runs, logger.Printf); err != nil { + logger.Printf("the build seats' ask-taken and step-progress are not heard, so a composition run is ended by no bound: %v", err) } } if inputs != nil { @@ -705,8 +712,13 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil { run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf} - steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, now: time.Now, logf: logger.Printf, + steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume, now: time.Now, logf: logger.Printf, asker: stepAskerOf(seatlist.Definition(gate.Reader, meshwants.Seat))} + // The state's watch wakes the steps waiting on a run and hears a bound's expiry; the runs standing at the start are taken up + // once the values that stand are read (§719, §720). + if err := watchRuns(ctx, kvs[buckets.CompositionRuns()], runs, runs.expired, func() { steps.takeUp(ctx, runs) }); err != nil { + return fmt.Errorf("the state %s is not watched: %w", buckets.CompositionRunsState, err) + } if err := serveCompositionSteps(ctx, nc, steps); err != nil { logger.Printf("the composition step is not heard, so no composition step runs until the assigner starts again: %v", err) } else { diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 6da96af1..b212850c 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -137,7 +137,7 @@ func TestManifest(t *testing.T) { // ADR 0618 §4: and it hears each node's engine say its reads changed; ADR 0636 §4 (issue 1966): and the seat list's two events; // issue 1966 note 151: and the teller's ask-ended, its subject the one the assigner hears; §376, §389: its own version-delivered and // the watchman's machine-away and machine-back, read whole at its start. - if !slices.Equal(m.Consumes, []string{"mesh-conditions.findings-asked", "mesh-conditions.condition-raised", "mesh-conditions.condition-changed", "mesh-conditions.condition-cleared", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "mesh-catalog.version-released", "mesh-wants.version-delivered", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", builds.BuilderModule + "." + builds.BuildsStreamEvents, buckets.IndexerModule + ".take-in.taken", buckets.TellerModule + "." + tellerevents.AskEnded, buckets.TellerModule + ".views." + buckets.AnswersState, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-build-go.step-ended", "node-engine.manifest-reads-changed", "node-engine.running", "mesh-delivery.pinned", "mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pins-stated"}) { + if !slices.Equal(m.Consumes, []string{"mesh-conditions.findings-asked", "mesh-conditions.condition-raised", "mesh-conditions.condition-changed", "mesh-conditions.condition-cleared", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "mesh-catalog.version-released", "mesh-wants.version-delivered", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", builds.BuilderModule + "." + builds.BuildsStreamEvents, buckets.IndexerModule + ".take-in.taken", buckets.TellerModule + "." + tellerevents.AskEnded, buckets.TellerModule + ".views." + buckets.AnswersState, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-build-go.step-ended", "node-build-go.ask-taken", "node-build-go.step-progress", "node-engine.manifest-reads-changed", "node-engine.running", "mesh-delivery.pinned", "mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pins-stated"}) { t.Errorf("consumes = %v, want the catalog half's revisions (design 47 G §195), the teller's ask-ended, the engines' reads changed and what they run", m.Consumes) } if want := []string{"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", @@ -147,7 +147,8 @@ func TestManifest(t *testing.T) { "seat:mesh-delivery.source", // §507: the branches the gate keeps a source revision of, before the first registrations "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone "seat:node-backup.backed-up", "seat:node-backup.now", - "seat:node-build-go.accept.check", // A-231: the composition run's ask on the Go build seat's queue check (accept.check.) + "seat:node-build-go.accept.check", // A-231 + "seat:node-build-go.drop", // §720, A-469s: a composition run past its bound drops its ask: the composition run's ask on the Go build seat's queue check (accept.check.) "seat:node-build-go.output", // A-254: a composition run's answer kept as the build seat's object "seat:node-engine.compose-candidate", // §354, A-226: each machine's composition input "seat:node-engine.manifest-reads", // ADR 0618 §3 diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 24b63223..fa24d09f 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -59,6 +59,8 @@ "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-build-go.step-ended", + "node-build-go.ask-taken", + "node-build-go.step-progress", "node-engine.manifest-reads-changed", "node-engine.running", "mesh-delivery.pinned", @@ -82,6 +84,7 @@ "seat:node-backup.backed-up", "seat:node-backup.now", "seat:node-build-go.output", + "seat:node-build-go.drop", "seat:node-engine.compose-candidate", "seat:node-engine.manifest-reads", "seat:operator-channel.asks",