diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go new file mode 100644 index 00000000..5c1b5902 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go @@ -0,0 +1,375 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "time" + + "github.com/nats-io/nats.go/jetstream" + + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// A composition run's bound (design 47 G §720; A-466n, A-466p, A-466q, A-466s, A-468b, A-469b, A-469s, A-469y, A-470): a run +// has one clock, the key `bound.` of composition-runs with its own time to live composition-step-bound, written at the +// build seat's ask-taken of the run's ask and written again at each step-progress of that ask; an ask still waiting in the queue +// has no clock but `untaken.`, written at the run's start with the time to live run-untaken-bound. Either's expiry, +// heard as the bus's event, ends the run: its ask dropped through the build seat's drop, the run ended could not run naming the +// ask and the bound, so every reader ends could not run with it. While the ask waits for a build node each reader is said waiting +// by a wait-held on mesh-wants (`on: build node`), ended `ran` at ask-taken or `superseded` when the run ends untaken. + +// waitOnBuildNode is what a composition run's waiting reader waits on (A-469b, A-469y). +const waitOnBuildNode = "build node" + +// dropVerbKey is the Go build seat's drop, asked as a kept call (§119, A-182). +const dropVerbKey = "seat:" + nodebuildgo.Seat + "." + nodebuildgo.VerbDrop + +// dropOver asks the Go build seat's drop of a run ask as a kept call, of any holder, which records it (A-469s). +func dropOver(call func(key string, body any) (json.RawMessage, error)) func(ask, why string) error { + return func(ask, why string) error { + raw, err := call(dropVerbKey, nodebuildgo.DropInput{ID: ask, Why: why, By: runAsker}) + if err != nil { + return err + } + if _, said, ok := buscalls.Unwrap(raw); !ok { + return fmt.Errorf("%s answered: %s", dropVerbKey, said) + } + return nil + } +} + +// waitOf is a reader's wait for a build node to take its run's ask (A-469y). +func waitOf(run meshwants.CompositionRun, reader meshwants.CompositionReader) meshwants.Wait { + w := meshwants.Wait{Step: "pipeline", Owner: meshwants.Seat, Module: reader.Module, Node: run.Machine, On: waitOnBuildNode, Fp: reader.Fingerprint, + Words: fmt.Sprintf("the composition step of %s waits for a build node to take the composition run %s of %s (ask %s)", reader.Pipeline, + runKeyOf(run.Candidate, run.Machine), run.Machine, run.Ask)} + if run.Started != nil { + w.Since = *run.Started + } + return w +} + +// sayWaits says, for each reader, its wait held or ended (how: ran, superseded); a wait that cannot be said is journaled. +func (r *compositionRuns) sayWaits(run meshwants.CompositionRun, readers []meshwants.CompositionReader, how string) { + if r.say == nil { + return + } + for _, c := range readers { + w := waitOf(run, c) + event, id := meshwants.EventWaitHeld, "composition."+runKeyOf(run.Candidate, run.Machine)+"."+run.Ask+"."+c.ID+".held" + if how != "" { + w.How, event, id = how, meshwants.EventWaitEnded, strings.TrimSuffix(id, ".held")+".ended" + } + if err := r.say(event, w, id); err != nil { + r.logf("the %s of %s on the composition run of %s is not said: %v", event, c.Pipeline, run.Machine, err) + } + } +} + +// untaken is whether the run of ask has not been taken yet: its untaken key stands holding that ask (A-470n). +func (r *compositionRuns) untaken(key, ask string) bool { + held, _, standing, err := r.boundOf(runUntakenPrefix + key) + return err == nil && standing && held == ask +} + +// boundOf reads a bound key: the ask it holds and its revision; standing false when it is not written. +func (r *compositionRuns) boundOf(boundKey string) (string, uint64, bool, error) { + raw, rev, err := r.kv.get(boundKey) + if err != nil || raw == nil { + return "", 0, false, err + } + var v boundValue + _ = json.Unmarshal(raw, &v) + return v.Ask, rev, true, nil +} + +// armBound writes the bound key prefix+key holding ask, judged from a read of the run, then reads the run again (A-470n): when +// it no longer names ask, or has ended, the bound just written is deleted at its revision, so no bound written from the earlier +// read ends the run after it, and a bound another run wrote since is kept. It answers whether the bound stands for the run. +func (r *compositionRuns) armBound(prefix, key, ask string, ttl time.Duration) (bool, error) { + rev, err := r.kv.arm(prefix+key, ask, ttl) + if err != nil { + return false, err + } + run, _, ok, err := r.read(key) + if err != nil { + return false, err + } + if ok && run.Ended == nil && run.Ask == ask { + return true, nil + } + if err := r.kv.delAt(prefix+key, rev); err != nil && !errors.Is(err, errRunChanged) { + return false, err + } + return false, nil +} + +// heardAskTaken begins a run's clock at its ask's ask-taken: the bound key armed holding the ask, the untaken key of that ask +// deleted, each reader's wait ended ran. An ask no running run names is another asker's, or a late signal of a run replaced or +// ended, which moves nothing and is journaled (A-470n). +func (r *compositionRuns) heardAskTaken(e nodebuildgo.AskTaken) error { + key, run, err := r.runOfAsk(e.Asker, e.Ask) + if err != nil { + return err + } + if key == "" { + r.late("ask-taken", e.Asker, e.Ask) + return nil + } + armed, err := r.armBound(runBoundPrefix, key, e.Ask, r.set.stepBound) + if err != nil { + return fmt.Errorf("the bound of the composition run %s is not armed: %w", key, err) + } + if !armed { + r.late("ask-taken", e.Asker, e.Ask) + return nil + } + if r.untaken(key, e.Ask) { + if err := r.clearBounds(key, e.Ask, runUntakenPrefix); err != nil { + return err + } + r.sayWaits(run, run.Readers, "ran") + } + return nil +} + +// heardProgress begins a running run's clock again at each step-progress of its ask (§720: owed progress, not run length); a +// late one moves nothing and is journaled (A-470n). +func (r *compositionRuns) heardProgress(e nodebuildgo.StepProgress) error { + key, _, err := r.runOfAsk(e.Asker, e.Ask) + if err != nil { + return err + } + armed := false + if key != "" { + if armed, err = r.armBound(runBoundPrefix, key, e.Ask, r.set.stepBound); err != nil { + return err + } + } + if !armed { + r.late("step-progress", e.Asker, e.Ask) + } + return nil +} + +// late journals a signal of an ask the assigner put whose run no key names running any more (A-470n). +func (r *compositionRuns) late(signal, asker, ask string) { + if asker == runAsker && ask != "" { + r.logf("the %s of the run ask %s is late: no composition run names it running, so it moves nothing (A-470n)", signal, ask) + } +} + +// runOfAsk is the running run of an ask the assigner put, or none. +func (r *compositionRuns) runOfAsk(asker, ask string) (string, meshwants.CompositionRun, error) { + if asker != runAsker || ask == "" { + return "", meshwants.CompositionRun{}, nil + } + key, run, err := r.keyOfAsk(ask) + if err != nil || key == "" || run.Ended != nil { + return "", meshwants.CompositionRun{}, err + } + return key, run, nil +} + +// expired ends the run whose bound key's time to live ended (§720): its ask dropped, the run ended could not run naming the +// ask and the bound. A run that ended already, or a key of none, takes nothing. +func (r *compositionRuns) expired(boundKey string) { + key, bound, setting := "", time.Duration(0), "" + switch { + case strings.HasPrefix(boundKey, runBoundPrefix): + key, bound, setting = strings.TrimPrefix(boundKey, runBoundPrefix), r.set.stepBound, "composition-step-bound" + case strings.HasPrefix(boundKey, runUntakenPrefix): + key, bound, setting = strings.TrimPrefix(boundKey, runUntakenPrefix), r.set.untaken, "run-untaken-bound" + default: + return + } + run, _, ok, err := r.read(key) + if err != nil || !ok || run.Ended != nil { + return + } + // The marker of an expiry holds no value (A-470o): the run ends only while its bound is still absent when read again, and + // the run is taken for bound., untaken for untaken., so an expiry heard after a bound was armed again, or of a run replaced + // since, ends nothing. + other := runUntakenPrefix + if setting == "run-untaken-bound" { + other = runBoundPrefix + } + _, _, armed, err := r.boundOf(boundKey) + if err != nil { + r.logf("the bound %s is not read again, so its expiry ends nothing: %v", boundKey, err) + return + } + held, _, standing, err := r.boundOf(other + key) + if err != nil { + r.logf("the bound %s is not read, so the expiry of %s ends nothing: %v", other+key, boundKey, err) + return + } + if armed || (standing && held == run.Ask) { + r.logf("the expiry of %s ends nothing: the bound is armed again, or the run of ask %s is not where it bounds (A-470o)", boundKey, run.Ask) + return + } + said := fmt.Sprintf("%s: its run ask %s said no progress within %s (%s)", run.Machine, run.Ask, bound, setting) + if setting == "run-untaken-bound" { + said = fmt.Sprintf("%s: its run ask %s was taken by no build node within %s (%s)", run.Machine, run.Ask, bound, setting) + } + if r.drop != nil { + if err := r.drop(run.Ask, said); err != nil { + r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err) + } + } + // The untaken key is gone with its expiry, so the run ended by it is said untaken here: its readers' waits end superseded. + if _, err := r.endRunOf(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil, setting == "run-untaken-bound"); err != nil { + r.logf("the composition run %s past its bound is not ended: %v", key, err) + } +} + +// endRun ends a run (end) and clears its bounds: the untaken key's readers said superseded, both bound keys deleted. +func (r *compositionRuns) endRun(key, ask, outcome, said string, verdict map[string]any) (bool, error) { + return r.endRunOf(key, ask, outcome, said, verdict, r.untaken(key, ask)) +} + +// endRunOf is endRun of a run whose ask is untaken, or not, as its caller judged. +func (r *compositionRuns) endRunOf(key, ask, outcome, said string, verdict map[string]any, untaken bool) (bool, error) { + return r.endRunMoving(key, ask, outcome, said, verdict, untaken, true) +} + +// endRunMoving is endRunOf, moving the readers a rerun carried or not: a drop says its readers' ends itself (A-470r). +func (r *compositionRuns) endRunMoving(key, ask, outcome, said string, verdict map[string]any, untaken, move bool) (bool, error) { + // The readers a rerun carried move with the run's end (§727, A-470d): chosen before the end is written, while the rerun's own + // step still waits on it, so it alone says its own end; each mover waits for the end in the key. + if moved := r.moved.Load(); moved != nil && move { + if run, _, ok, err := r.read(key); err == nil && ok && run.Ended == nil && run.Ask == ask { + (*moved)(run) + } + } + ended, err := r.end(key, ask, outcome, said, verdict) + if err != nil || !ended { + return ended, err + } + if untaken { + if run, _, ok, err := r.read(key); err == nil && ok { + r.sayWaits(run, run.Readers, "superseded") + } + } + if err := r.clearBounds(key, ask, runBoundPrefix, runUntakenPrefix); err != nil { + r.logf("the bound keys of the ended composition run %s are not deleted: %v", key, err) + } + return true, nil +} + +// readersOfOpen are the readers of every run with no `ended`, each with its ask's id once: what the assigner takes up again at +// its start (§719), each reader's end said from the keys alone (A-468). A reader whose step said its end already, carried into a +// rerun's run, is taken up only when it is marked `rerun` (A-470l, A-470m); each with the ask of the open run it is taken up from. +func (r *compositionRuns) readersOfOpen() ([]openReader, error) { + all, err := r.keys() + if err != nil { + return nil, err + } + seen := map[string]bool{} + var out []openReader + for _, key := range all { + if !isRunKey(key) { + continue + } + run, _, ok, err := r.read(key) + if err != nil || !ok || run.Ended != nil { + continue + } + for _, c := range run.Readers { + if c.Ended != nil && !c.Rerun { + continue // it ended passed already: only a reader marked rerun says its end again + } + if !seen[c.ID] { + seen[c.ID] = true + out = append(out, openReader{CompositionReader: c, ask: run.Ask}) + } + } + } + return out, nil +} + +// openReader is a reader taken up at the start and the ask of the open run it is taken up from. +type openReader struct { + meshwants.CompositionReader + ask string +} + +// keysOf are the keys of the candidate's runs that the reader reads. +func (r *compositionRuns) keysOf(candidate, id string) ([]string, error) { + all, err := r.keys() + if err != nil { + return nil, err + } + var out []string + prefix := keysPart(candidate) + "." + for _, key := range all { + if !isRunKey(key) || !strings.HasPrefix(key, prefix) { + continue + } + if run, _, ok, err := r.read(key); err == nil && ok && hasReader(run.Readers, id) { + out = append(out, key) + } + } + slices.Sort(out) // by machine, as a step names its machines + return out, nil +} + +// hearRunBounds hears the build seats' ask-taken and step-progress on the assigner's durable consumer (§720, A-466q): each +// handed to the runs; one that cannot be written is given again by the bus. +func hearRunBounds(hub interface { + Handle(string, func(jetstream.Msg)) error +}, r *compositionRuns, logf func(string, ...any)) error { + if err := hub.Handle(nodebuildgo.SubjectAskTakenAll, func(m jetstream.Msg) { + var e nodebuildgo.AskTaken + if err := json.Unmarshal(m.Data(), &e); err != nil { + logf("%s holds what is no ask-taken, left: %v", m.Subject(), err) + _ = m.Ack() + return + } + ackOrNak(m, r.heardAskTaken(e), logf) + }); err != nil { + return err + } + return hub.Handle(nodebuildgo.SubjectStepProgressAll, func(m jetstream.Msg) { + var e nodebuildgo.StepProgress + if err := json.Unmarshal(m.Data(), &e); err != nil { + logf("%s holds what is no step-progress, left: %v", m.Subject(), err) + _ = m.Ack() + return + } + ackOrNak(m, r.heardProgress(e), logf) + }) +} + +func ackOrNak(m jetstream.Msg, err error, logf func(string, ...any)) { + if err != nil { + logf("%s is not taken, so the bus gives it again: %v", m.Subject(), err) + _ = m.Nak() + return + } + _ = m.Ack() +} + +// takeUp says, at the assigner's start, the end of each reader of a run with no `ended` once its runs end (§719, A-468): the +// step's goroutine is gone, so its end is judged from the candidate's keys naming it. +func (c *compositionSteps) takeUp(ctx context.Context, r *compositionRuns) { + readers, err := r.readersOfOpen() + if err != nil { + c.logf("composition-runs: the runs standing at the start are not read, so their readers are not taken up: %v", err) + return + } + for _, reader := range readers { + c.logf("the composition step %s of %s is taken up again by its runs (ask %s)", reader.Pipeline, reader.Module, reader.ID) + again := "" // a reader whose step said its end already says it again under the open run's ask (A-470q) + if reader.Ended != nil { + again = reader.ask + } + go c.say1(ctx, reader.ID, stepRunOf(reader.CompositionReader), c.resume, again) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go index 2e763747..52c52263 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go @@ -10,7 +10,6 @@ import ( "errors" "fmt" "slices" - "sync" "time" nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" @@ -18,8 +17,8 @@ import ( meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" ) -// The inputs of a composition step (design 47 G §354, A-243, A-252): one request per machine, each carrying that machine's input, -// asked of the machine's node-engine by the seat's verb compose-candidate. +// The inputs of a composition run (design 47 G §354, §718, A-243, A-252): one request per candidate and machine, carrying that +// machine's input, asked of the machine's node-engine by the seat's verb compose-candidate when the run starts. // composeCandidateKey is the seat verb's call key, asked of one node. const composeCandidateKey = "seat:" + ne.Seat + "." + ne.VerbComposeCandidate @@ -80,37 +79,14 @@ type compositionInputs struct { bound time.Duration } -// gather asks each machine of the step's module and branch for its input with the step's candidate key, and answers every machine in -// name order: one away (away) is not asked; a node that is silent, unready or answers no input has Why set; no machine at all is an -// error. The run keeps each request built from an input under its run ask (compositionRun.machine). -func (c *compositionInputs) gather(ctx context.Context, run meshwants.StepRun, away func(node string) bool) ([]nodeInput, error) { - key, err := candidateKeyOf(run) - if err != nil { - return nil, err - } - nodes := c.machines(run.Module, run.Branch) - if len(nodes) == 0 { - return nil, fmt.Errorf("no node places %s on the branch %q, so the composition step has no machine to compose", run.Module, run.Branch) - } +// of asks the machine node for its input with the candidate's key (§718): asked only by the composition step that starts the +// machine's composition run, once per candidate and machine. A node that is silent, unready or answers no input has Why set. +func (c *compositionInputs) of(ctx context.Context, node, candidate string) nodeInput { bound := c.bound if bound <= 0 { bound = nec.AnswerBound } - out := make([]nodeInput, len(nodes)) - var wg sync.WaitGroup - for i, node := range nodes { - if away != nil && away(node) { - out[i] = nodeInput{Node: node, Away: true} - continue - } - wg.Add(1) - go func() { - defer wg.Done() - out[i] = c.one(ctx, bound, node, key) - }() - } - wg.Wait() - return out, nil + return c.one(ctx, bound, node, candidate) } // one asks one node for its input. diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go index 78fe48e1..d403de52 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go @@ -68,7 +68,7 @@ func TestTheMachinesOfAStep(t *testing.T) { // §354, A-243, A-252, §64: each machine is asked compose-candidate with the step's key and input true, its numbers as written; a // silent or unready node says why, and a machine away is not asked. func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) { - // gather asks the machines side by side, so the record of what each was asked is guarded. + // the machines are asked side by side, so the record of what each was asked is guarded. var mu sync.Mutex asked := map[string]string{} call := func(key string, body any) (json.RawMessage, error) { @@ -86,7 +86,7 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) { return json.RawMessage(`{"candidate":"run-r1","node":"` + node + `","verdict":"composed","input":{"n":12345678901234567890,"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil } c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14", "novox", "shanks"} }, callFile: call, bound: time.Second} - got, err := c.gather(context.Background(), meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" }) + got, err := gatherAll(c, meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" }) if err != nil || len(got) != 4 { t.Fatalf("%+v %v", got, err) } @@ -105,7 +105,7 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) { t.Errorf("the machine away is asked or not said away: %+v", got[3]) } none := &compositionInputs{machines: func(string, string) []string { return nil }, callFile: call} - if _, err := none.gather(context.Background(), meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil { + if _, err := gatherAll(none, meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil { t.Error("a step with no machine is gathered") } } @@ -124,7 +124,7 @@ func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) { return json.RawMessage(answer + `,"file":"` + file + `"}`), nil } c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14"} }, callFile: call, bound: time.Second} - got, err := c.gather(context.Background(), meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(string) bool { return false }) + got, err := gatherAll(c, meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(string) bool { return false }) if err != nil || len(got) != 2 { t.Fatalf("%+v %v", got, err) } @@ -135,3 +135,28 @@ func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) { t.Errorf("g14: %+v", got[1]) } } + +// gatherAll asks each machine of a step for its input as each run's start does (compositionInputs.of), side by side; a machine away +// is not asked; no machine is an error. +func gatherAll(c *compositionInputs, run meshwants.StepRun, away func(string) bool) ([]nodeInput, error) { + key, err := candidateKeyOf(run) + if err != nil { + return nil, err + } + nodes := c.machines(run.Module, run.Branch) + if len(nodes) == 0 { + return nil, errors.New("no machine") + } + out := make([]nodeInput, len(nodes)) + var wg sync.WaitGroup + for i, node := range nodes { + if away != nil && away(node) { + out[i] = nodeInput{Node: node, Away: true} + continue + } + wg.Add(1) + go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key) }() + } + wg.Wait() + return out, nil +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go index 3422319c..0bf7224e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go @@ -21,12 +21,14 @@ import ( "git.novox.be/novox/mesh-sdk/go/keys" ) -// The run of a composition step (design 47 G §354, §63, §64; A-226, A-238, A-240, A-243, A-252, A-254, A-261): one -// composer.Request per machine, depth compose, the one machine and one case `change`, its manifests, verdicts and seats from the -// step's catalog revision read through mesh-catalog's `revision`; one run ask per machine of the seat node-build-go, naming no -// head, its program the node-engine build's catalog entry with the composer's package; the run's end heard on the build seat's -// step-ended of that ask, its answer (a composer.Verdict) read inline from `output`. A machine the watchman said away -// (machine-away) is not composed and passes with a warning (§64). +// The run of a composition step (design 47 G §354, §63, §64, §718; A-226, A-238, A-240, A-252, A-254, A-261): the step reads one +// composition run per machine of its candidate (compositionruns.go), and the step that starts a machine's run composes it: one +// compose-candidate ask of the node, one composer.Request, depth compose, the one machine and one case `change`, its manifests, +// verdicts and seats from the step's catalog revision read through mesh-catalog's `revision`, kept once in `inputs`; one run ask +// of the seat node-build-go, naming no head, its program the node-engine build's catalog entry with the composer's package, +// chosen once and recorded in the run. The run's end is heard on the build seat's step-ended of that ask, its answer (a +// composer.Verdict) read from `output`, and written in the run's key, whose change every reading step hears. A machine the +// watchman said away (machine-away) is not composed and passes with a warning (§64). const ( // runAsker is the assigner's name as an asker of a build seat, its seat's own name: the last token of the subject its run @@ -41,42 +43,6 @@ const ( seatDefinitionsKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbSeatDefinitions ) -// runEnds hands each build seat's step-ended of an ask to the composition run that waits for it (A-240), by the ask's id. -type runEnds struct { - mu sync.Mutex - waits map[string]chan buildstep.StepEnded -} - -// expect is the end of the ask id, once heard; called before the ask is put, so no end is missed. -func (r *runEnds) expect(id string) <-chan buildstep.StepEnded { - r.mu.Lock() - defer r.mu.Unlock() - if r.waits == nil { - r.waits = map[string]chan buildstep.StepEnded{} - } - ch := make(chan buildstep.StepEnded, 1) - r.waits[id] = ch - return ch -} - -// forget stops waiting for the ask id. -func (r *runEnds) forget(id string) { - r.mu.Lock() - delete(r.waits, id) - r.mu.Unlock() -} - -// heard hands an end to the run that waits for its ask; one nobody waits for is another asker's, left. -func (r *runEnds) heard(e buildstep.StepEnded) { - r.mu.Lock() - ch := r.waits[e.Ask] - delete(r.waits, e.Ask) - r.mu.Unlock() - if ch != nil { - ch <- e - } -} - // awayNodes are the machines the watchman said away (machine-away of the seat mesh-watchdogs, §389) and not back since. type awayNodes struct { mu sync.Mutex @@ -109,6 +75,8 @@ func (a *awayNodes) is(machine string) bool { // compositionRun runs a composition step. type compositionRun struct { + // runs are the composition runs of the state composition-runs (compositionruns.go). + runs *compositionRuns inputs *compositionInputs keeper *inputKeeper // call asks a seat verb held once for the mesh (process.Conn.CallKept). @@ -119,8 +87,7 @@ type compositionRun struct { running func(node, module string) (ne.Running, bool) away *awayNodes // put puts a run ask on the build seat's check queue under the assigner's name. - put func(ctx context.Context, a buildstep.StepAsk) error - ends *runEnds + put func(ctx context.Context, a buildstep.StepAsk) error // reads shares the catalog reads among steps (compositionreads.go); sleep waits between two tries of a read (nil: a timer). reads *compositionReads sleep func(ctx context.Context, d time.Duration) error @@ -132,46 +99,81 @@ type compositionRun struct { // machineEnd is what one machine came to. type machineEnd struct { - failed, couldNot, warning string + failed, couldNot string + warnings []string // judged is set where the composer judged the machine, composing or not (A-467a). judged bool } -// run runs one composition step and answers what it came to: failed when a machine does not compose, could not run when a machine -// could not be composed or was not judged, else passed with a warning per machine away. -func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRun) stepOutcome { - half, err := r.revision(ctx, run) - if err != nil { - return couldNotRun("the step's catalog revision is not read: " + err.Error()) - } - seats, err := r.seats(ctx, half.Revision.Definitions) +// run runs one composition step and answers what it came to: per machine not away it joins or starts the candidate's +// composition run (§718), waits for each run's end in its key, and judges each machine from it; failed when a machine does not +// compose, could not run when a machine could not be composed or was not judged, else passed with a warning per machine away. +func (r *compositionRun) run(ctx context.Context, id string, step meshwants.StepRun) stepOutcome { + candidate, err := candidateKeyOf(step) if err != nil { return couldNotRun(err.Error()) } - manifests, verdicts, err := caseOf(half) - if err != nil { - return couldNotRun(err.Error()) + nodes := r.inputs.machines(step.Module, step.Branch) + if len(nodes) == 0 { + return couldNotRun(fmt.Sprintf("no node places %s on the branch %q, so the composition step has no machine to compose", step.Module, step.Branch)) } - programOf, err := r.programs(ctx, half) - if err != nil { - return couldNotRun(err.Error()) - } - inputs, err := r.inputs.gather(ctx, run, r.away.is) - if err != nil { - return couldNotRun(err.Error()) + reader := readerOf(id, step) + prep := &runPrep{r: r, step: step} + var ends []machineEnd + var keys []string + for _, node := range nodes { + if r.away.is(node) { + ends = append(ends, machineEnd{warnings: []string{node + " is away (the watchman's machine-away), so it is not composed"}}) + continue + } + join := r.runs.join + if step.Rerun { // a step asked again (§727) + join = r.runs.rejoin + } + j, err := join(reader, candidate, node, buildstep.NewID()) + if err != nil { + ends = append(ends, machineEnd{couldNot: node + ": " + err.Error()}) + continue + } + if j.started { + go r.start(ctx, j.key, j.run, prep) + } + keys = append(keys, j.key) } + return r.read(ctx, step.Module, keys, ends) +} + +// read waits for the end of each run of keys and judges the step of module from them, naming in its progress and its end the +// runs it reads and their asks (§723, §729). +func (r *compositionRun) read(ctx context.Context, module string, keys []string, ends []machineEnd) stepOutcome { + setShared(ctx, r.runs.sharedOf(keys, nil)) setPhase(ctx, meshwants.StepProgressEventPhaseRun) - ends := make([]machineEnd, len(inputs)) - var wg sync.WaitGroup - for i, in := range inputs { - wg.Add(1) - go func() { - defer wg.Done() - ends[i] = r.machine(ctx, in, buildstep.NewID(), programOf, manifests, verdicts, seats) - }() - } - wg.Wait() + runs := r.runs.await(ctx, keys) setPhase(ctx, meshwants.StepProgressEventPhaseReport) + for _, key := range keys { + ends = append(ends, judgeRun(module, runs[key])) + } + out := outcomeOf(ends) + out.Shared = r.runs.sharedOf(keys, runs) + return out +} + +// resume judges a step taken up at the assigner's start (§719, A-468): the runs of its candidate naming it, once each has ended. +func (r *compositionRun) resume(ctx context.Context, id string, step meshwants.StepRun) stepOutcome { + candidate, err := candidateKeyOf(step) + if err != nil { + return couldNotRun(err.Error()) + } + keys, err := r.runs.keysOf(candidate, id) + if err != nil { + return couldNotRun("the step's composition runs are not read: " + err.Error()) + } + return r.read(ctx, step.Module, keys, nil) +} + +// outcomeOf is a step's outcome from its machines: failed before could not run before passed (§354); a step that judged no +// machine never passes (A-467a), each machine not judged named with why. +func outcomeOf(ends []machineEnd) stepOutcome { var out stepOutcome var failed, couldNot []string judged := 0 @@ -181,14 +183,14 @@ func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRu } failed = appendSome(failed, e.failed) couldNot = appendSome(couldNot, e.couldNot) - out.Warnings = appendSome(out.Warnings, e.warning) + out.Warnings = append(out.Warnings, e.warnings...) } switch { case len(failed) > 0: out.Outcome, out.Said = meshwants.StepEndedOutcomeFailed, strings.Join(failed, "; ") case len(couldNot) > 0: out.Outcome, out.Said = meshwants.StepEndedOutcomeCouldNotRun, strings.Join(couldNot, "; ") - case judged == 0: // a step that judged no machine never passes (A-467a): each machine not judged is named with why + case judged == 0: said := "no machine was judged" if len(out.Warnings) == 0 { said += ": the step has no machine" @@ -213,82 +215,187 @@ func couldNotRun(said string) stepOutcome { return stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: said} } -// machine composes one machine: its request kept under its ask, the ask put, its end heard and its verdict read. -func (r *compositionRun) machine(ctx context.Context, in nodeInput, ask string, programOf func(node string) (buildstep.RunProgram, error), - manifests map[string]composer.Manifest, verdicts map[string]json.RawMessage, seats []json.RawMessage) machineEnd { - node := in.Node - switch { - case in.Away: - return machineEnd{warning: node + " is away (the watchman's machine-away), so it is not composed"} - case in.Why != "": - return machineEnd{couldNot: node + " gave no composition input: " + in.Why} +// runPrep is what the step that starts a run reads of its catalog revision, once for every machine it starts: the seats, the +// case's manifests and verdicts, and the composer program per machine. +type runPrep struct { + r *compositionRun + step meshwants.StepRun + once sync.Once + // read once + seats []json.RawMessage + manifests map[string]composer.Manifest + verdicts map[string]json.RawMessage + none map[string]string + programOf func(node string) (buildstep.RunProgram, string, error) + err error +} + +func (p *runPrep) get(ctx context.Context) error { + p.once.Do(func() { + half, err := p.r.revision(ctx, p.step) + if err != nil { + p.err = errors.New("the step's catalog revision is not read: " + err.Error()) + return + } + if p.seats, p.err = p.r.seats(ctx, half.Revision.Definitions); p.err != nil { + return + } + if p.manifests, p.verdicts, p.none, p.err = caseOf(half); p.err != nil { + return + } + p.programOf, p.err = p.r.programs(ctx, half) + }) + return p.err +} + +// start composes the machine of the run it started (§718): the node's input of this moment, its request kept once in `inputs`, +// the composer program chosen and recorded with the request's digest, and one run ask put. A fault ends the run could not run. +func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep) { + node, ask := run.Machine, run.Ask + end := func(said string) { + if _, err := r.runs.endRun(key, ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil); err != nil { + r.logf("the composition run %s is not ended: %v", key, err) + } } - program, err := programOf(node) + if err := prep.get(ctx); err != nil { + end(node + ": " + err.Error()) + return + } + in := r.inputs.of(ctx, node, run.Candidate) + if in.Why != "" { + end(node + " gave no composition input: " + in.Why) + return + } + program, engineFp, err := prep.programOf(node) if err != nil { - return machineEnd{couldNot: node + ": " + err.Error()} + end(node + ": " + err.Error()) + return } req := composer.Request{V: composer.Version, Input: in.Body, Depth: composer.DepthCompose, Machines: []string{node}, - Cases: []composer.Case{{Name: changeCase, Manifests: manifests, Verdicts: verdicts}}, Seats: seats} + Cases: []composer.Case{{Name: changeCase, Manifests: prep.manifests, Verdicts: prep.verdicts, None: prep.none}}, Seats: prep.seats, + BlameTries: r.runs.set.blameTries} // the composer's bound of its search for culprits (§721, A-466r) raw, err := json.Marshal(req) if err != nil { - return machineEnd{couldNot: node + ": its request cannot be encoded: " + err.Error()} + end(node + ": its request cannot be encoded: " + err.Error()) + return } - if _, err := r.keeper.keep(ask, raw); err != nil { + digest, err := r.keeper.keep(ask, raw) + if err != nil { r.logf("the composition request of %s is refused: %v", node, err) - return machineEnd{couldNot: node + ": its request is not kept: " + err.Error()} + end(node + ": its request is not kept: " + err.Error()) + return } ref, err := buildstep.RunInputOf(inputVerbKey, raw) if err != nil { - return machineEnd{couldNot: node + ": " + err.Error()} + end(node + ": " + err.Error()) + return + } + if _, _, err := r.runs.change(key, func(c *meshwants.CompositionRun) bool { + if c.Ask != ask || c.Ended != nil { + return false + } + c.Program = &meshwants.CompositionRunProgram{Repository: program.Repository, Commit: program.Commit, Directory: program.Dir, Fingerprint: engineFp} + c.Input = digest + return true + }); err != nil { + end(node + ": the run's program and input are not written: " + err.Error()) + return } a := buildstep.StepAsk{ID: ask, Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: runAsker, At: r.now().UTC(), Run: &buildstep.RunJob{Program: program, Heads: []buildstep.RunHead{}, Input: ref, Reader: "the composition of " + node + " by the node-engine at " + program.Commit}} - ended := r.ends.expect(ask) - defer r.ends.forget(ask) if err := r.put(ctx, a); err != nil { - return machineEnd{couldNot: node + ": its run ask " + ask + " is not put: " + err.Error()} + end(node + ": its run ask " + ask + " is not put: " + err.Error()) } - var e buildstep.StepEnded - select { - case <-ctx.Done(): - return machineEnd{couldNot: node + ": the step stopped before its run ask " + ask + " ended"} - case e = <-ended: - } - if e.Outcome != buildstep.OutcomePassed { - return machineEnd{couldNot: fmt.Sprintf("%s: its run ask %s ended %s on %s: %s", node, ask, e.Outcome, e.Node, e.Said)} - } - answer, err := e.ReadOutput(buildstep.AskGo, r.ask) // the program's last line of standard output (A-254) - if err != nil { - return machineEnd{couldNot: node + ": " + err.Error()} - } - return verdictOf(node, answer) } -// verdictOf reads the composer's verdict of one machine: not composing is failed, naming why; not judged is could not run, naming why (A-467). -func verdictOf(node string, answer []byte) machineEnd { +// notTried is the problem the composer names a module with that its search for culprits did not try, past the request's +// blame_tries (design 47 G §721; the composer's composer.NotTried in mesh-host, compared by its words, A-470b). +const notTried = "not tried: the search reached its bound" + +// judgeRun judges one machine of the step of module from the run it read (§721, §722): could not run with the run's words; else +// from the machine's composer verdict: not judged is could not run (A-467); a verdict blaming the step's module fails it, or is +// could not run where the module was not tried (A-470b); a machine that does not compose and blames no module fails every step on +// it, on the node's own account; a verdict with no `blamed` (a composer before §721) is judged as before, with the warning "the +// composer names no culprits"; otherwise the machine passes, with a warning for each other module blamed there. +func judgeRun(module string, run meshwants.CompositionRun) machineEnd { + node := run.Machine + if run.Ended == nil { + return machineEnd{couldNot: node + ": the step stopped before its composition run " + runKeyOf(run.Candidate, node) + " ended"} + } + if run.Outcome != meshwants.CompositionRunOutcomePassed { + return machineEnd{couldNot: run.Said} + } + m, err := machineOf(run.Verdict) + if err != nil { + return machineEnd{couldNot: node + ": the run's verdict does not read: " + err.Error()} + } + if m.NotJudged != "" { // a step never passes on a machine it did not judge (A-467) + return machineEnd{couldNot: node + " was not judged: " + m.NotJudged} + } + why := strings.Join(appendSome(append(slices.Clone(m.Problems), m.Refusals...), m.Reason), "; ") + switch { + case m.Blamed == nil && !m.Composes: // an old composer (§721) + return machineEnd{judged: true, failed: node + " does not compose: " + why, warnings: []string{node + ": the composer names no culprits"}} + case len(m.Blamed) == 0 && !m.Composes: // the node's own account (A-452) + return machineEnd{judged: true, failed: node + " does not compose and blames no module: " + why} + } + end := machineEnd{judged: true} + var own, others []string + for _, b := range m.Blamed { + switch { + case b.Module == module && b.Problem == notTried: + end.couldNot = fmt.Sprintf("%s holds %s untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", node, module) + case b.Module == module: + own = append(own, b.Problem) + default: + others = append(others, node+" holds "+b.Module+": "+b.Problem) + } + } + if len(own) > 0 { + end.failed, end.couldNot = node+" holds "+module+": "+strings.Join(own, "; "), "" + } + end.warnings = others + return end +} + +// machineOf is the composer's verdict of one machine as a run keeps it. +func machineOf(v map[string]any) (composer.Machine, error) { + var m composer.Machine + raw, err := json.Marshal(v) + if err == nil { + err = json.Unmarshal(raw, &m) + } + return m, err +} + +// runVerdict reads the composer's answer of one machine: its verdict as the run keeps it, or why it is none. +func runVerdict(node string, answer []byte) (map[string]any, string) { var v composer.Verdict if err := json.Unmarshal(answer, &v); err != nil { - return machineEnd{couldNot: node + ": the composer answered no verdict: " + err.Error()} + return nil, node + ": the composer answered no verdict: " + err.Error() } if v.V != composer.Version { - return machineEnd{couldNot: fmt.Sprintf("%s: the composer answered a verdict of version %d, the assigner reads %d", node, v.V, composer.Version)} + return nil, fmt.Sprintf("%s: the composer answered a verdict of version %d, the assigner reads %d", node, v.V, composer.Version) } for _, c := range v.Cases { if c.Name != changeCase { continue } m, ok := c.Machines[node] - switch { - case !ok: - return machineEnd{couldNot: node + ": the composer's verdict names no such machine"} - case m.NotJudged != "": // a step never passes on a machine it did not judge (A-467) - return machineEnd{couldNot: node + " was not judged: " + m.NotJudged} - case !m.Composes: - return machineEnd{judged: true, failed: node + " does not compose: " + strings.Join(appendSome(append(slices.Clone(m.Problems), m.Refusals...), m.Reason), "; ")} + if !ok { + return nil, node + ": the composer's verdict names no such machine" } - return machineEnd{judged: true} + raw, err := json.Marshal(m) + var out map[string]any + if err == nil { + err = json.Unmarshal(raw, &out) + } + if err != nil { + return nil, node + ": the composer's verdict cannot be kept: " + err.Error() + } + return out, "" } - return machineEnd{couldNot: node + ": the composer's verdict has no case " + changeCase} + return nil, node + ": the composer's verdict has no case " + changeCase } // revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252), once for every step of @@ -400,20 +507,28 @@ func (r *compositionRun) seatAt(ctx context.Context, seat, digest string) (json. return def, nil } -// caseOf is the case's manifests and verdicts from the revision's entries: each manifest whole, and the verdict of each one not -// accepted (a module without one is accepted). -func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMessage, error) { +// caseOf is the case's manifests, verdicts and none from the revision's entries: each manifest whole, the verdict of each one not +// accepted (a module without one is accepted), and the revision's none words of each module it gives no fingerprint, whose +// build can never be made (design 47 G A-470j, A-470k). +func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMessage, map[string]string, error) { manifests := map[string]composer.Manifest{} verdicts := map[string]json.RawMessage{} + var none map[string]string for name, e := range half.Entries { if len(e.Manifest) == 0 { continue } raw, err := json.Marshal(e.Manifest) if err != nil { - return nil, nil, fmt.Errorf("the manifest of %s cannot be encoded: %w", name, err) + return nil, nil, nil, fmt.Errorf("the manifest of %s cannot be encoded: %w", name, err) } manifests[name] = composer.Manifest{Raw: raw} + if e.Fp == "" && e.None != "" { + if none == nil { + none = map[string]string{} + } + none[name] = e.None + } if e.Verdict == "" || e.Verdict == mcc.VerdictAccepted { continue } @@ -426,11 +541,11 @@ func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMes } raw, err = json.Marshal(v) if err != nil { - return nil, nil, err + return nil, nil, nil, err } verdicts[name] = raw } - return manifests, verdicts, nil + return manifests, verdicts, none, nil } // nodeEngineEntry is the entry of the module that claims the seat node-engine, or false. @@ -468,50 +583,53 @@ func programOfEntry(e meshcatalog.CatalogEntry) (buildstep.RunProgram, error) { return buildstep.RunProgram{Repository: e.Repository, Commit: e.Commit, Dir: dir, Package: composer.Package}, nil } -// programs answers, per machine, the composer its run builds (A-226, A-243, A-252): where the revision's node-engine entry differs -// from the trunk's, that build for every machine; else the build the machine's node-engine runs, as its `running` says, its entry -// read through mesh-catalog's `entry`; a machine whose node-engine's `running` is not heard gets the revision's entry, said (A-451). -func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, error), error) { +// programs answers, per machine, the composer its run builds and the node-engine fingerprint it was chosen from (A-226, A-243, +// A-252, §719): where the revision's node-engine entry differs from the trunk's, that build for every machine; else the build the +// machine's node-engine runs, as its `running` says, its entry read through mesh-catalog's `entry`; a machine whose node-engine's +// `running` is not heard gets the revision's entry, said (A-451). +func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, string, error), error) { engine, ok := nodeEngineEntry(half.Entries) if !ok { return nil, fmt.Errorf("the catalog revision %d holds no module claiming the seat %s", half.Revision.Revision, ne.Seat) } if trunk, ok := r.trunk(); !ok || func() bool { t, ok := nodeEngineEntry(trunk.Entries); return !ok || t.Fp != engine.Fp }() { p, err := programOfEntry(engine) - return func(string) (buildstep.RunProgram, error) { return p, err }, nil + return func(string) (buildstep.RunProgram, string, error) { return p, engine.Fp, err }, nil } var mu sync.Mutex byFp := map[string]buildstep.RunProgram{} - return func(node string) (buildstep.RunProgram, error) { + return func(node string) (buildstep.RunProgram, string, error) { running, ok := r.running(node, engine.Module) if !ok || running.Fingerprint == "" { r.logf("%s: no build of %s is heard running there, so its composer is the catalog revision's entry %s (A-451)", node, engine.Module, engine.Fp) - return programOfEntry(engine) + p, err := programOfEntry(engine) + return p, engine.Fp, err } + fp := running.Fingerprint mu.Lock() - p, ok := byFp[running.Fingerprint] + p, ok := byFp[fp] mu.Unlock() if ok { - return p, nil + return p, fp, nil } - what := "the entry of the node-engine's build " + running.Fingerprint + " is not read" - body, said, err := r.callRetry(ctx, entryVerbKey, meshcatalog.EntryInput{Fingerprint: running.Fingerprint}) + what := "the entry of the node-engine's build " + fp + " is not read" + body, said, err := r.callRetry(ctx, entryVerbKey, meshcatalog.EntryInput{Fingerprint: fp}) if err != nil || body == nil { - return buildstep.RunProgram{}, unreadErr(what, err, said) + return buildstep.RunProgram{}, fp, unreadErr(what, err, said) } var a meshcatalog.EntryAnswer if err := json.Unmarshal(body, &a); err != nil { - return buildstep.RunProgram{}, fmt.Errorf("%s: %w", what, err) + return buildstep.RunProgram{}, fp, fmt.Errorf("%s: %w", what, err) } p, err = programOfEntry(a.Entry) if err != nil { - return p, err + return p, fp, err } mu.Lock() - byFp[running.Fingerprint] = p + byFp[fp] = p mu.Unlock() - return p, nil + return p, fp, nil }, nil } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go index d64e621e..a7cc38e6 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "slices" "strings" "sync" "testing" @@ -30,12 +31,16 @@ type runStand struct { requests map[string]composer.Request // by node entries []string // the fingerprints `entry` was asked r *compositionRun + runs *compositionRuns + puts int // the run asks put // noDefinitions: the revision names no seat definition (A-234's stand-in); trunkSeats: the trunk's seat list as (seat, digest) // pairs; unjudged: the machine the composer does not judge; listFault: the trunk's seat list cannot be read. noDefinitions bool trunkSeats []meshcatalog.SeatDefinitionsAnswerSeatsItem unjudged string listFault error + // verdict, when set, is the composer's verdict of a machine (by node), else the stand's own. + verdict func(node string) (composer.Machine, bool) } func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *runStand { @@ -52,7 +57,8 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) * "": {Revision: &meshcatalog.CatalogRevision{Revision: 7, Branch: "main", Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "d1"}}}, Entries: []meshcatalog.CatalogEntry{engine(revisionEngine, revisionCommit), {Module: "bad", Verdict: mcc.VerdictRefused, Inputs: &meshcatalog.EntryInputs{Problems: []string{"no seat x"}}, Manifest: map[string]any{"module": "bad"}}}, Next: "bad"}, - "bad": {Revision: &meshcatalog.CatalogRevision{Revision: 7}, Entries: []meshcatalog.CatalogEntry{{Module: "m", Verdict: mcc.VerdictAccepted, Manifest: map[string]any{"module": "m"}}}}, + "bad": {Revision: &meshcatalog.CatalogRevision{Revision: 7}, Entries: []meshcatalog.CatalogEntry{{Module: "m", Verdict: mcc.VerdictAccepted, Manifest: map[string]any{"module": "m"}}, + {Module: "lidarr", None: "builders disagree: node", Manifest: map[string]any{"module": "lidarr"}}}}, // no fingerprint (A-470j) } call := func(key string, body any) (json.RawMessage, error) { raw, _ := json.Marshal(body) @@ -99,10 +105,10 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) * away := &awayNodes{} away.set("shanks", true) keeper := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now) - ends := &runEnds{} - s.r = &compositionRun{ + s.runs = newCompositionRuns(newRunsMem(), defaultRunSettings, time.Now, t.Logf) + s.r = &compositionRun{runs: s.runs, inputs: &compositionInputs{machines: func(string, string) []string { return []string{"ace", "g14", "shanks"} }, callFile: engines, bound: time.Second}, - keeper: keeper, reads: &compositionReads{}, call: call, ends: ends, away: away, now: time.Now, logf: t.Logf, + keeper: keeper, reads: &compositionReads{}, call: call, away: away, now: time.Now, logf: t.Logf, trunk: func() (mcc.Half, bool) { return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{"engine": engine(trunkEngine, commitE1)}}, true }, @@ -124,13 +130,27 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) * s.asks[node], s.requests[node] = a, req s.mu.Unlock() m := composer.Machine{Composes: true} - if node == s.unjudged { + override, set := composer.Machine{}, false + if s.verdict != nil { + override, set = s.verdict(node) + } + if set { + m = override + } else if node == s.unjudged { m = composer.Machine{NotJudged: "the composer was handed no seat list"} } else if node == "g14" { m = composer.Machine{Reason: "two modules own one path", Problems: []string{"/etc/x is m's and bad's"}} } v, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: changeCase, Machines: map[string]composer.Machine{node: m}}}}) - go ends.heard(buildstep.StepEnded{Ask: a.ID, Asker: runAsker, Node: "builder1", Outcome: buildstep.OutcomePassed, Output: string(v)}) + s.mu.Lock() + s.puts++ + s.mu.Unlock() + go func() { + if err := s.runs.heardEnd(buildstep.StepEnded{Ask: a.ID, Asker: runAsker, Node: "builder1", Outcome: buildstep.OutcomePassed, Output: string(v)}, + func(e buildstep.StepEnded) ([]byte, error) { return []byte(e.Output), nil }); err != nil { + t.Error(err) + } + }() return nil }, } @@ -146,7 +166,8 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) { s := newRunStand(t, "src2:e1", "src2:e1", "d1") out := s.r.run(context.Background(), "ask-step", step) if out.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(out.Said, "g14 does not compose") || !strings.Contains(out.Said, "/etc/x") || - strings.Contains(out.Said, "ace") || len(out.Warnings) != 1 || !strings.Contains(out.Warnings[0], "shanks is away") { + strings.Contains(out.Said, "ace") || len(out.Warnings) != 2 || !strings.Contains(out.Warnings[0], "shanks is away") || + out.Warnings[1] != "g14: the composer names no culprits" { t.Fatalf("the step came to %+v", out) } if len(s.asks) != 2 { @@ -160,12 +181,12 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) { if s.asks["g14"].Run.Program.Commit != commitE2 { t.Errorf("g14's composer is built at %s, want the build its node-engine runs", s.asks["g14"].Run.Program.Commit) } - if req.Depth != composer.DepthCompose || len(req.Machines) != 1 || req.Machines[0] != "ace" || len(req.Cases) != 1 || req.Cases[0].Name != changeCase || + if req.Depth != composer.DepthCompose || req.BlameTries != 20 || len(req.Machines) != 1 || req.Machines[0] != "ace" || len(req.Cases) != 1 || req.Cases[0].Name != changeCase || string(req.Input) != `{"node":"ace"}` || len(req.Seats) != 1 || string(req.Seats[0]) != `{"seat":"s1"}` { t.Errorf("the request of ace %+v", req) } c := req.Cases[0] - if string(c.Manifests["m"].Raw) != `{"module":"m"}` || len(c.Manifests) != 3 || len(c.Verdicts) != 1 || !strings.Contains(string(c.Verdicts["bad"]), `"refused"`) || + if string(c.Manifests["m"].Raw) != `{"module":"m"}` || len(c.Manifests) != 4 || len(c.None) != 1 || c.None["lidarr"] != "builders disagree: node" || len(c.Verdicts) != 1 || !strings.Contains(string(c.Verdicts["bad"]), `"refused"`) || !strings.Contains(string(c.Verdicts["bad"]), "no seat x") { t.Errorf("the case %+v %s", c.Manifests, c.Verdicts) } @@ -329,3 +350,227 @@ func TestTheCompositionStepReadsTheRevisionOnceAndAgainOnATimeout(t *testing.T) }) } } + +// Design 47 G §718, §719, §720: three composition steps of one revision on two machines start two runs, one request and one run +// ask per machine, and each step reads both verdicts; the run records its program with the node-engine fingerprint it was chosen +// from and the digest of its kept request, which names no module of the asking pipeline; a step of another candidate starts its +// own runs, and a run that could not run is never read again. +func TestTheStepsOfOneCandidateReadOneRunPerMachine(t *testing.T) { + s := newRunStand(t, "src2:e1", "src2:e1", "d1") + s.r.inputs.machines = func(string, string) []string { return []string{"ace", "g14"} } + var wg sync.WaitGroup + outs := make([]stepOutcome, 3) + for i, m := range []string{"m", "m2", "m3"} { + wg.Add(1) + go func() { + defer wg.Done() + outs[i] = s.r.run(context.Background(), "ask-"+m, meshwants.StepRun{Pipeline: "main.7." + m, Module: m, Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}) + }() + } + wg.Wait() + for i, o := range outs { + if len(o.Shared) != 2 || o.Shared[0].Machine != "ace" || o.Shared[0].Run != runKeyOf("main.7", "ace") || o.Shared[0].Ask == "" || + o.Shared[1].Machine != "g14" || o.Shared[1].Ask == o.Shared[0].Ask { // §723, §729: per machine the run read and its ask + t.Errorf("step %d names the runs %+v", i, o.Shared) + } + if o.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(o.Said, "g14 does not compose") { + t.Errorf("step %d came to %+v, want both machines read", i, o) + } + } + if s.puts != 2 { + t.Fatalf("%d run asks put, want one per machine", s.puts) + } + ace, _, _, _ := s.runs.read(runKeyOf("main.7", "ace")) + g14, _, _, _ := s.runs.read(runKeyOf("main.7", "g14")) + if len(ace.Readers) != 3 || len(g14.Readers) != 3 || ace.Program == nil || ace.Program.Fingerprint != "src2:e1" || ace.Program.Commit != commitE1 || + g14.Program == nil || g14.Program.Fingerprint != "src2:e2" || ace.Input == "" || ace.Input != digestOf(mustJSON(t, s.requests["ace"])) || + ace.Outcome != meshwants.CompositionRunOutcomePassed || g14.Outcome != meshwants.CompositionRunOutcomePassed || g14.Verdict["composes"] != false { + t.Errorf("the runs: ace %+v, g14 %+v", ace, g14) + } + if req := s.requests["ace"]; strings.Contains(string(mustJSON(t, req)), "main.7.m") || string(req.Input) != `{"node":"ace"}` { + t.Errorf("the request names the asking pipeline or not the node's input: %s", mustJSON(t, req)) + } + // Another candidate (a run on request) starts its own runs. + if o := s.r.run(context.Background(), "ask-r", meshwants.StepRun{Pipeline: "run-r1.m", Module: "m", Branch: "main", Revision: 7, Run: "r1", Step: meshwants.StepRunStepComposition}); o.Outcome != meshwants.StepEndedOutcomeFailed || s.puts != 4 { + t.Errorf("a step of another candidate came to %+v with %d asks, want its own two runs", o, s.puts) + } + // A run whose ask could not be put could not run; the next step of the candidate starts it again. + f := newRunStand(t, "src2:e1", "src2:e1", "d1") + f.r.inputs.machines = func(string, string) []string { return []string{"ace"} } + put, refused := f.r.put, 0 + f.r.put = func(ctx context.Context, a buildstep.StepAsk) error { + if refused == 0 { + refused++ + return errors.New("no stream") + } + return put(ctx, a) + } + first := f.r.run(context.Background(), "ask-1", meshwants.StepRun{Pipeline: "main.7.m", Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}) + second := f.r.run(context.Background(), "ask-2", meshwants.StepRun{Pipeline: "main.7.m2", Module: "m2", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}) + if first.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(first.Said, "is not put") || second.Outcome != meshwants.StepEndedOutcomePassed || f.puts != 1 { + t.Errorf("after a run that could not run: %+v then %+v, %d asks", first, second, f.puts) + } +} + +func mustJSON(t *testing.T, v any) []byte { + t.Helper() + raw, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return raw +} + +// Design 47 G §721, §722, A-470b: a step judges each machine from its verdict's `blamed`: a verdict blaming the step's module fails +// it and passes another module's step with a warning naming the module; a machine that does not compose and blames no module fails +// every step; a verdict with no `blamed` is judged as before and says "the composer names no culprits"; a module the search did not +// try is could not run for its own step; a run that could not run is could not run. +func TestAMachineIsJudgedByTheModulesItBlames(t *testing.T) { + at := time.Now() + runOf := func(verdict string) meshwants.CompositionRun { + var v map[string]any + if err := json.Unmarshal([]byte(verdict), &v); err != nil { + t.Fatal(err) + } + return meshwants.CompositionRun{Candidate: "main.7", Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomePassed, Verdict: v} + } + blamesA := `{"composes":false,"reason":"two modules own one path","blamed":[{"module":"a","problem":"/etc/x is a's and c's"}]}` + for _, tc := range []struct { + name, module string + run meshwants.CompositionRun + failed, couldNo string + warnings []string + }{ + {"blamed module fails", "a", runOf(blamesA), "ace holds a: /etc/x is a's and c's", "", nil}, + {"another module passes with a warning", "b", runOf(blamesA), "", "", []string{"ace holds a: /etc/x is a's and c's"}}, + {"blames no module fails both", "b", runOf(`{"composes":false,"reason":"the node's own composition","blamed":[]}`), "ace does not compose and blames no module: the node's own composition", "", nil}, + {"an old composer", "b", runOf(`{"composes":false,"reason":"x"}`), "ace does not compose: x", "", []string{"ace: the composer names no culprits"}}, + {"composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "", nil}, + {"own module not tried", "a", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "", + "ace holds a untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", nil}, + {"another not tried", "b", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "", "", + []string{"ace holds a: not tried: the search reached its bound"}}, + {"not judged", "a", runOf(`{"composes":false,"not_judged":"no seat list","blamed":[]}`), "", "ace was not judged: no seat list", nil}, + {"could not run", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"}, "", "ace: gone", nil}, + } { + t.Run(tc.name, func(t *testing.T) { + e := judgeRun(tc.module, tc.run) + if e.failed != tc.failed || e.couldNot != tc.couldNo || !slices.Equal(e.warnings, tc.warnings) { + t.Errorf("judged %+v, want failed %q, could not run %q, warnings %q", e, tc.failed, tc.couldNo, tc.warnings) + } + }) + } +} + +// Design 47 G §727, A-468g, A-470d: a rerun of a composition step starts a new run only on the machine it did not pass, replacing +// that key and carrying every former reader, the one that did not pass marked rerun with the rerun's caller; at the new run's end +// that reader says its end again, marked rerun, naming the new run's ask; a reader that passed does not move; the machine the +// step passed keeps its verdict. +func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) { + s := newRunStand(t, "src2:e1", "src2:e1", "d1") + s.r.inputs.machines = func(string, string) []string { return []string{"ace", "g14"} } + var mu sync.Mutex + fixed := false + s.verdict = func(node string) (composer.Machine, bool) { + mu.Lock() + defer mu.Unlock() + m := composer.Machine{Composes: true} + m.Blamed = []struct { + Module string `json:"module"` + Problem string `json:"problem"` + }{} + if node == "g14" && !fixed { + m.Composes, m.Reason = false, "two modules own one path" + m.Blamed = append(m.Blamed, struct { + Module string `json:"module"` + Problem string `json:"problem"` + }{"a", "/etc/x"}, struct { + Module string `json:"module"` + Problem string `json:"problem"` + }{"d", "/etc/x"}) + } + return m, true + } + ends := make(chan meshwants.StepEnded, 16) + msgIDs := map[string]string{} // the message id of each step's latest end + steps := &compositionSteps{node: "n1", now: time.Now, logf: t.Logf, run: s.r.run, resume: s.r.resume, ended: s.runs.stepEnded, + say: func(event string, body any, msgID string) error { + if e, ok := body.(meshwants.StepEnded); ok { + mu.Lock() + msgIDs[e.ID] = msgID + mu.Unlock() + ends <- e + } + return nil + }} + move := steps.move(context.Background()) + s.runs.moved.Store(&move) + step := func(m string, rerun bool) []byte { + raw, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.7." + m, Module: m, Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition, Rerun: rerun, By: map[bool]string{true: "op@n1"}[rerun]}) + return raw + } + got := map[string]meshwants.StepEnded{} + for _, m := range []string{"a", "b", "d"} { + steps.take(context.Background(), "ask-"+m, step(m, false)) + e := <-ends + got[e.Module] = e + } + if got["a"].Outcome != meshwants.StepEndedOutcomeFailed || got["d"].Outcome != meshwants.StepEndedOutcomeFailed || got["b"].Outcome != meshwants.StepEndedOutcomePassed { + t.Fatalf("the first ends: %+v", got) + } + former := got["d"].Shared[1].Ask + mu.Lock() + fixed = true + mu.Unlock() + steps.take(context.Background(), "ask-a2", step("a", true)) + moved := map[string]meshwants.StepEnded{} + for range 2 { + select { + case e := <-ends: + moved[e.ID] = e + case <-time.After(5 * time.Second): + t.Fatalf("the ends after the rerun: %+v", moved) + } + } + a, d := moved["ask-a2"], moved["ask-d"] + if a.Outcome != meshwants.StepEndedOutcomePassed || d.Outcome != meshwants.StepEndedOutcomePassed || !d.Rerun || d.By != "op@n1" || + len(d.Shared) != 2 || d.Shared[1].Machine != "g14" || d.Shared[1].Ask == former || d.Shared[1].Ask != a.Shared[1].Ask { + t.Fatalf("after the rerun: a %+v, d %+v", a, d) + } + mu.Lock() + // A-470q: the first end keeps the ask's id; the end said again names the moved run's ask, so the bus does not drop it. + if msgIDs["ask-a2"] != "ask-a2" || msgIDs["ask-d"] != "ask-d."+a.Shared[1].Ask || msgIDs["ask-b"] != "ask-b" { + t.Errorf("the message ids of the ends: %v", msgIDs) + } + mu.Unlock() + if s.puts != 3 { + t.Errorf("%d run asks, want one more for the machine a did not pass", s.puts) + } + select { + case e := <-ends: + t.Errorf("a reader that passed moved: %+v", e) + case <-time.After(50 * time.Millisecond): + } + run, _, _, _ := s.runs.read(runKeyOf("main.7", "g14")) + marks := map[string]bool{} + for _, c := range run.Readers { + marks[c.ID] = c.Rerun + } + if len(run.Readers) != 3 || !marks["ask-d"] || marks["ask-b"] || !marks["ask-a2"] { + t.Errorf("the new run's readers %+v", run.Readers) + } + // A-470m: each reader's `ended` is written once its step said its end, and ask-b's carried into the new run. + deadline := time.After(5 * time.Second) + for { + wake := s.runs.wake() + run, _, _, _ = s.runs.read(runKeyOf("main.7", "g14")) + if !slices.ContainsFunc(run.Readers, func(c meshwants.CompositionReader) bool { return c.Ended == nil }) { + break + } + select { + case <-wake: + case <-deadline: + t.Fatalf("a reader of the new run names no end: %+v", run.Readers) + } + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go new file mode 100644 index 00000000..62e59090 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go @@ -0,0 +1,896 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/nats-io/nats.go/jetstream" + + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/keys" + + "mesh-delivery-assigner/internal/buckets" +) + +// The composition runs (design 47 G §718, §719; A-468, A-469d, A-469w, A-469x): the assigner composes each machine once per +// candidate, whatever number of composition steps name it. Each run is one key of its old state `composition-runs`, +// `.`, each part escaped as a key's name; the step that starts a run writes the key create-only, +// so two steps of one candidate starting at once start one run and the second reads it. A reader is the composition step's +// StepRun with its ask's id (A-468), added and removed by compare-and-set. A key is deleted once composition-runs-kept has +// passed since its `ended`, judged at each new run over at most 20 of the oldest ended keys, as the state inputs is +// (A-465b, A-468b); at the cap (composition-runs-cap-mib, the manifest's max-mib its ceiling, A-469d) no run is started. + +const ( + // runsSwept is how many of the oldest ended keys each new run judges. + runsSwept = 20 + // runBoundPrefix and runUntakenPrefix open a run's two bound keys, each with its own time to live (§720, A-468b, + // A-470): `bound.` from the ask's ask-taken, `untaken.` from the run's start. + runBoundPrefix = "bound." + runUntakenPrefix = "untaken." +) + +// runKeyOf is the key of a candidate's composition run of a machine (§719). +func runKeyOf(candidate, machine string) string { + return keysPart(candidate) + "." + keys.EscapeName(machine) +} + +// keysPart is a name as a key's part. +func keysPart(name string) string { return keys.EscapeName(name) } + +// isRunKey is whether a key of the state is a run's own, not one of its bound keys. +func isRunKey(key string) bool { + return !strings.HasPrefix(key, runBoundPrefix) && !strings.HasPrefix(key, runUntakenPrefix) +} + +// runSettings are the settings of the composition runs, each read at the start from the file the manifest declares. +type runSettings struct { + // capBytes is composition-runs-cap-mib in bytes. + capBytes int64 + // kept is composition-runs-kept, stepBound composition-step-bound, untaken run-untaken-bound (A-469x). + kept, stepBound, untaken time.Duration + // blameTries is blame-tries, carried in each run's request (§721). + blameTries int +} + +// defaultRunSettings are the settings' defaults, as the manifest gives them. +var defaultRunSettings = runSettings{capBytes: 16 << 20, kept: 24 * time.Hour, stepBound: 30 * time.Minute, untaken: 24 * time.Hour, blameTries: 20} + +// runSettingsOf reads the settings composition-runs-cap-mib, composition-runs-kept, composition-step-bound, +// run-untaken-bound and blame-tries from the files the environment names (A-469x, as outsideSettings): a missing or empty +// file is its default; a content that is no positive whole number or Go duration is an error naming the file. +func runSettingsOf(getenv func(string) string) (runSettings, error) { + s := defaultRunSettings + mib, err := settingFile(getenv("MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE")) + if err != nil { + return s, err + } + if mib > 0 { + s.capBytes = int64(mib) << 20 + } + tries, err := settingFile(getenv("MESH_ASSIGNER_BLAME_TRIES_FILE")) + if err != nil { + return s, err + } + if tries > 0 { + s.blameTries = tries + } + for env, to := range map[string]*time.Duration{"MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": &s.kept, + "MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": &s.stepBound, "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": &s.untaken} { + d, err := durationFile(getenv(env)) + if err != nil { + return s, err + } + if d > 0 { + *to = d + } + } + return s, nil +} + +// durationFile reads one setting's file holding a Go duration: 0 for no path, no file or an empty one. +func durationFile(path string) (time.Duration, error) { + if path == "" { + return 0, nil + } + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return 0, nil + } + if err != nil { + return 0, err + } + v := strings.TrimSpace(string(raw)) + if v == "" { + return 0, nil + } + d, err := time.ParseDuration(v) + if err != nil || d <= 0 { + return 0, fmt.Errorf("%s holds %q, which is not a positive Go duration such as 30m", path, v) + } + return d, nil +} + +// errRunChanged is a compare-and-set of a run's key that met another write. +var errRunChanged = errors.New("the composition run's key changed") + +// runsKV is the state composition-runs: the bus bucket, or a test's. +type runsKV interface { + // get is a key's value and revision; nil for a key not there. + get(key string) ([]byte, uint64, error) + // create writes a key that does not stand, update one at its revision: errRunChanged when it does not stand so. + create(key string, value []byte) (uint64, error) + update(key string, value []byte, revision uint64) (uint64, error) + // delAt deletes a key at its revision: errRunChanged when it no longer stands so, nothing for a key not there. No write + // of the state is made from a value read before its latest change. + delAt(key string, revision uint64) error + keys() ([]string, error) + // arm writes a bound key afresh holding the ask of the run it bounds, with its own time to live (§720, A-470n): at the key's + // latest revision, a delete marker counted, read again when another write came between; its expiry is the bus's event. It + // answers the revision written. + arm(key, ask string, ttl time.Duration) (uint64, error) + // used is how many bytes the state holds. + used() (int64, error) +} + +// busRuns is runsKV over the bus bucket. +type busRuns struct { + ctx context.Context + js jetstream.JetStream + kv jetstream.KeyValue +} + +func (b busRuns) get(key string) ([]byte, uint64, error) { + e, err := b.kv.Get(b.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, 0, nil + } + if err != nil { + return nil, 0, err + } + return e.Value(), e.Revision(), nil +} + +func (b busRuns) create(key string, value []byte) (uint64, error) { + rev, err := b.kv.Create(b.ctx, key, value) + if errors.Is(err, jetstream.ErrKeyExists) { + return 0, errRunChanged + } + return rev, err +} + +func (b busRuns) update(key string, value []byte, revision uint64) (uint64, error) { + rev, err := b.kv.Update(b.ctx, key, value, revision) + if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + return 0, errRunChanged + } + return rev, err +} + +func (b busRuns) delAt(key string, revision uint64) error { + err := b.kv.Delete(b.ctx, key, jetstream.LastRevision(revision)) + if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + return errRunChanged + } + return err +} + +func (b busRuns) keys() ([]string, error) { + l, err := b.kv.ListKeys(b.ctx) + if err != nil { + return nil, err + } + var out []string + for k := range l.Keys() { + out = append(out, k) + } + return out, nil +} + +// boundValue is a bound key's value: the key and the ask of the run it bounds (A-470n). +type boundValue struct { + Bound string `json:"bound"` + Ask string `json:"ask"` +} + +// arm publishes the bound key holding ask with ttl as its own time to live, expecting the key's latest message (a delete +// marker counted), so a write between is read again and never overwritten unseen (A-470n); the earlier message, replaced, +// takes its time to live with it, so the expiry is heard only ttl after the last write. +func (b busRuns) arm(key, ask string, ttl time.Duration) (uint64, error) { + if b.js == nil { + return 0, errors.New("no bus to arm a bound on") + } + raw, err := json.Marshal(boundValue{Bound: key, Ask: ask}) + if err != nil { + return 0, err + } + subject := "$KV." + b.kv.Bucket() + "." + key + for { + st, err := b.js.Stream(b.ctx, "KV_"+b.kv.Bucket()) + if err != nil { + return 0, err + } + var last uint64 + m, err := st.GetLastMsgForSubject(b.ctx, subject) + switch { + case err == nil: + last = m.Sequence + case !errors.Is(err, jetstream.ErrMsgNotFound): + return 0, err + } + ack, err := b.js.Publish(b.ctx, subject, raw, jetstream.WithMsgTTL(ttl), jetstream.WithExpectLastSequencePerSubject(last)) + if err != nil && strings.Contains(err.Error(), "wrong last sequence") { + continue + } + if err != nil { + return 0, err + } + return ack.Sequence, nil + } +} + +func (b busRuns) used() (int64, error) { + st, err := b.kv.Status(b.ctx) + if err != nil { + return 0, err + } + return int64(st.Bytes()), nil +} + +// compositionRuns keeps the composition runs in the state composition-runs. +type compositionRuns struct { + kv runsKV + set runSettings + now func() time.Time + logf func(string, ...any) + mu sync.Mutex + // changed is closed and replaced at each change of the state heard (heard): a step waiting on a run reads its key again. + changed chan struct{} + // seen are the keys the state's watch handed and has not seen deleted: a listing of the bucket misses a key that is + // being written at that moment, so every lookup takes these beside it (keys). + seen map[string]bool + // say says an event of the seat mesh-wants (a reader's wait, A-469y); drop asks the build seat's drop of a run ask (§119); + // nil says or drops nothing. + say publisher + drop func(ask, why string) error + // moved is told each run that ended, for the readers a rerun carried to say their end again (compositionSteps.move). + // Set once the steps are served, after the hub starts. + moved atomic.Pointer[func(run meshwants.CompositionRun)] +} + +func newCompositionRuns(kv runsKV, set runSettings, now func() time.Time, logf func(string, ...any)) *compositionRuns { + return &compositionRuns{kv: kv, set: set, now: now, logf: logf, changed: make(chan struct{})} +} + +// heard takes a change of the state from its watch (a module watches its own state, principle 1). +func (r *compositionRuns) heard() { + r.mu.Lock() + close(r.changed) + r.changed = make(chan struct{}) + r.mu.Unlock() +} + +// saw keeps a key the state's watch handed: standing, or deleted or expired. +func (r *compositionRuns) saw(key string, standing bool) { + r.mu.Lock() + defer r.mu.Unlock() + if r.seen == nil { + r.seen = map[string]bool{} + } + if standing { + r.seen[key] = true + } else { + delete(r.seen, key) + } +} + +// keys are the state's keys: the bucket's listing with the keys its watch handed beside it, sorted. A key read gone later +// is skipped by its reader. +func (r *compositionRuns) keys() ([]string, error) { + all, err := r.kv.keys() + if err != nil { + return nil, err + } + r.mu.Lock() + for key := range r.seen { + all = append(all, key) + } + r.mu.Unlock() + slices.Sort(all) + return slices.Compact(all), nil +} + +// wake is closed at the next change heard. +func (r *compositionRuns) wake() <-chan struct{} { + r.mu.Lock() + defer r.mu.Unlock() + return r.changed +} + +// read is a run's value and revision, or false. +func (r *compositionRuns) read(key string) (meshwants.CompositionRun, uint64, bool, error) { + raw, rev, err := r.kv.get(key) + if err != nil || raw == nil { + return meshwants.CompositionRun{}, 0, false, err + } + var run meshwants.CompositionRun + if err := json.Unmarshal(raw, &run); err != nil { + return meshwants.CompositionRun{}, 0, false, fmt.Errorf("the composition run %s does not read: %w", key, err) + } + return run, rev, true, nil +} + +// change writes a run's key by compare-and-set: f changes the value read and says whether to write it; false leaves it. +// It answers the value written or left, and false for a key not there. +func (r *compositionRuns) change(key string, f func(*meshwants.CompositionRun) bool) (meshwants.CompositionRun, bool, error) { + for { + run, rev, ok, err := r.read(key) + if err != nil || !ok { + return run, ok, err + } + if !f(&run) { + return run, true, nil + } + raw, err := json.Marshal(run) + if err != nil { + return run, true, err + } + if _, err := r.kv.update(key, raw, rev); errors.Is(err, errRunChanged) { + continue + } else if err != nil { + return run, true, err + } + r.heard() + return run, true, nil + } +} + +// fill is the state's fill against its cap. +func (r *compositionRuns) fill() (meshwants.Fill, error) { + at := r.now().UTC() + used, err := r.kv.used() + return meshwants.Fill{Store: buckets.CompositionRunsState, Bytes: used, Cap: r.set.capBytes, At: &at}, err +} + +// joined is a reader's place in a run: the run's key, whether this reader started it (so it composes the machine), and the +// value as it stood. +type joined struct { + key string + started bool + run meshwants.CompositionRun +} + +// join finds or starts the candidate's run of the machine for reader (§718, §719, §720): a run that stands and has not +// ended, or that ended other than could not run, is read, the reader added; none, or one that could not run, is started: +// written create-only (or replacing the ended key by compare-and-set), the reader its first, ask its run ask's id. At the +// cap no run is started, an error naming the fill. +func (r *compositionRuns) join(reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) { + key := runKeyOf(candidate, machine) + for { + run, rev, ok, err := r.read(key) + if err != nil { + return joined{}, err + } + if ok && (run.Ended == nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun) { + if hasReader(run.Readers, reader.ID) { + return joined{key: key, run: run}, nil + } + run.Readers = append(run.Readers, reader) + raw, err := json.Marshal(run) + if err != nil { + return joined{}, err + } + if _, err := r.kv.update(key, raw, rev); errors.Is(err, errRunChanged) { + continue + } else if err != nil { + return joined{}, err + } + r.heard() + if run.Ended == nil && r.untaken(key, run.Ask) { + r.sayWaits(run, []meshwants.CompositionReader{reader}, "") + } + return joined{key: key, run: run}, nil + } + if err := r.room(); err != nil { + return joined{}, err + } + started := r.now().UTC() + fresh := meshwants.CompositionRun{Candidate: candidate, Machine: machine, Ask: ask, Started: &started, + Readers: []meshwants.CompositionReader{reader}} + raw, err := json.Marshal(fresh) + if err != nil { + return joined{}, err + } + if ok { + _, err = r.kv.update(key, raw, rev) // could not run is never read: a new run replaces it (§720) + } else { + _, err = r.kv.create(key, raw) + } + if errors.Is(err, errRunChanged) { + continue + } else if err != nil { + return joined{}, err + } + r.heard() + // An ask waiting in the queue has no clock but run-untaken-bound from its start (A-466s, A-468b, A-470, A-470n). + if _, err := r.armBound(runUntakenPrefix, key, ask, r.set.untaken); err != nil { + r.logf("the untaken bound of the composition run %s is not armed: %v", key, err) + } + r.sayWaits(fresh, fresh.Readers, "") + return joined{key: key, started: true, run: fresh}, nil + } +} + +// room sweeps the state and says whether a new run may start: an error naming the fill at the cap (as A-465a). +func (r *compositionRuns) room() error { + if err := r.sweep(); err != nil { + r.logf("composition-runs: the keys past composition-runs-kept are not all deleted: %v", err) + } + f, err := r.fill() + if err != nil { + return fmt.Errorf("the fill of the state %s is not read, so no composition run is started: %w", buckets.CompositionRunsState, err) + } + if f.Bytes >= f.Cap { + return fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so no composition run is started", buckets.CompositionRunsState, + f.Cap>>20, f.Bytes) + } + return nil +} + +// sweep deletes, among the 20 oldest ended runs, those ended more than composition-runs-kept ago, with their bound keys. +func (r *compositionRuns) sweep() error { + all, err := r.keys() + if err != nil { + return err + } + type endedAt struct { + key string + at time.Time + } + var ended []endedAt + for _, key := range all { + if !isRunKey(key) { + continue + } + run, _, ok, err := r.read(key) + if err != nil || !ok || run.Ended == nil { + continue + } + ended = append(ended, endedAt{key, *run.Ended}) + } + slices.SortFunc(ended, func(a, b endedAt) int { return a.at.Compare(b.at) }) + if len(ended) > runsSwept { + ended = ended[:runsSwept] + } + var errs []error + for _, e := range ended { + if r.now().Sub(e.at) >= r.set.kept { + errs = append(errs, r.deleteRun(e.key, func(run meshwants.CompositionRun) bool { + return run.Ended != nil && r.now().Sub(*run.Ended) >= r.set.kept + })) + } + } + return errors.Join(errs...) +} + +// deleteRun deletes a run's key with its bound keys while the value it holds is still one gone judges gone, each delete at the +// revision just read and judged again on a conflict: a run that replaced the key meanwhile, and its bounds, stand. +func (r *compositionRuns) deleteRun(key string, gone func(meshwants.CompositionRun) bool) error { + for { + run, rev, ok, err := r.read(key) + if err != nil || !ok || !gone(run) { + return err + } + if err := r.clearBounds(key, run.Ask, runBoundPrefix, runUntakenPrefix); err != nil { + return err + } + if err := r.kv.delAt(key, rev); errors.Is(err, errRunChanged) { + continue + } else if err != nil { + return err + } + r.heard() + return nil + } +} + +// clearBounds deletes the bound keys of the run of ask while key holds that run (or none) and the bound holds that ask: each at +// the revision it was read at, read again on a conflict, so a bound a newer run of the key armed is never deleted. +func (r *compositionRuns) clearBounds(key, ask string, prefixes ...string) error { + var errs []error + for _, prefix := range prefixes { + for { + run, _, ok, err := r.read(key) + if err != nil { + errs = append(errs, err) + break + } + if ok && run.Ask != ask { + break // a newer run holds the key: its bounds are its own + } + held, rev, standing, err := r.boundOf(prefix + key) + if err != nil || !standing { + errs = append(errs, err) + break + } + if held != ask { + break // the bound of another ask (A-470n): not this run's to delete + } + if err := r.kv.delAt(prefix+key, rev); errors.Is(err, errRunChanged) { + continue + } else if err != nil { + errs = append(errs, err) + } + break + } + } + return errors.Join(errs...) +} + +// stepEnded writes at as the `ended` of the reader asked under id in each run of its step's candidate naming it, by +// compare-and-set (A-470m): the time its step said its end, carried with it into a rerun's run. +func (r *compositionRuns) stepEnded(id string, step meshwants.StepRun, at time.Time) { + candidate, err := candidateKeyOf(step) + if err != nil { + r.logf("the end of the composition step %s is not written on its runs: %v", id, err) + return + } + keys, err := r.keysOf(candidate, id) + if err != nil { + r.logf("the end of the composition step %s is not written on its runs: %v", id, err) + return + } + at = at.UTC() + for _, key := range keys { + if _, _, err := r.change(key, func(c *meshwants.CompositionRun) bool { + i := slices.IndexFunc(c.Readers, func(c meshwants.CompositionReader) bool { return c.ID == id }) + if i < 0 { + return false + } + c.Readers[i].Ended = &at + return true + }); err != nil { + r.logf("the end of the composition step %s is not written on the composition run %s: %v", id, key, err) + } + } +} + +// hasReader is whether readers holds the step asked under id. +func hasReader(readers []meshwants.CompositionReader, id string) bool { + return slices.ContainsFunc(readers, func(c meshwants.CompositionReader) bool { return c.ID == id }) +} + +// readerOf is a composition step as a run's reader: its StepRun with its ask's id (A-468). +func readerOf(id string, s meshwants.StepRun) meshwants.CompositionReader { + return meshwants.CompositionReader{ID: id, Pipeline: s.Pipeline, Step: s.Step, Module: s.Module, Branch: s.Branch, Commit: s.Commit, + Revision: s.Revision, Source: s.Source, Fingerprint: s.Fingerprint, Run: s.Run, Rerun: s.Rerun, By: s.By} +} + +// stepRunOf is a reader's StepRun again, for its step-ended. +func stepRunOf(c meshwants.CompositionReader) meshwants.StepRun { + return meshwants.StepRun{Pipeline: c.Pipeline, Step: c.Step, Module: c.Module, Branch: c.Branch, Commit: c.Commit, Revision: c.Revision, + Source: c.Source, Fingerprint: c.Fingerprint, Run: c.Run, Rerun: c.Rerun, By: c.By} +} + +// watchRuns hears the changes of the state composition-runs (principle 1): each change wakes the steps waiting on a run, and +// the expiry of a bound key, the bus's marker, is handed to expired (§720). After the values that stand, ready is called. +func watchRuns(ctx context.Context, kv jetstream.KeyValue, r *compositionRuns, expired func(key string), ready func()) error { + w, err := kv.WatchAll(ctx) + if err != nil { + return err + } + go func() { + defer func() { _ = w.Stop() }() + for { + select { + case <-ctx.Done(): + return + case e, ok := <-w.Updates(): + if !ok { + return + } + if e == nil { + if ready != nil { + ready() + ready = nil + } + continue + } + r.saw(e.Key(), e.Operation() == jetstream.KeyValuePut) + if e.Operation() == jetstream.KeyValuePurge && !isRunKey(e.Key()) && expired != nil { + expired(e.Key()) + } + r.heard() + } + } + }() + return nil +} + +// end ends the run of ask in its key with an outcome, its words and the machine's verdict, by compare-and-set; it answers +// false when the key holds another run or the run has ended already. +func (r *compositionRuns) end(key, ask, outcome, said string, verdict map[string]any) (bool, error) { + ended := false + _, _, err := r.change(key, func(c *meshwants.CompositionRun) bool { + if c.Ask != ask || c.Ended != nil { + return false + } + at := r.now().UTC() + c.Ended, c.Outcome, c.Said, c.Verdict = &at, outcome, said, verdict + ended = true + return true + }) + return ended, err +} + +// keyOfAsk is the key of the run whose ask is ask, or none. +func (r *compositionRuns) keyOfAsk(ask string) (string, meshwants.CompositionRun, error) { + all, err := r.keys() + if err != nil { + return "", meshwants.CompositionRun{}, err + } + for _, key := range all { + if !isRunKey(key) { + continue + } + run, _, ok, err := r.read(key) + if err == nil && ok && run.Ask == ask { + return key, run, nil + } + } + return "", meshwants.CompositionRun{}, nil +} + +// heardEnd ends the run of the ask a build seat's step-ended names (§719, A-240): a run ask that ended other than passed +// could not run; a passed one's answer, read with read, is the machine's verdict, or why it is none. An end of an ask no run +// names is another asker's, or of a run gone, and changes nothing. +func (r *compositionRuns) heardEnd(e buildstep.StepEnded, read func(buildstep.StepEnded) ([]byte, error)) error { + if e.Asker != runAsker || e.Ask == "" { + return nil + } + key, run, err := r.keyOfAsk(e.Ask) + if err != nil || key == "" { + return err + } + if run.Ended != nil { // ended by its bound or a drop: the end heard later is ignored (§720) + r.logf("the step-ended of the run ask %s is ignored: its composition run %s ended before it (%s)", e.Ask, key, run.Said) + return nil + } + node := run.Machine + outcome, said := meshwants.CompositionRunOutcomeCouldNotRun, "" + var verdict map[string]any + if e.Outcome != buildstep.OutcomePassed { + said = fmt.Sprintf("%s: its run ask %s ended %s on %s: %s", node, e.Ask, e.Outcome, e.Node, e.Said) + } else if answer, err := read(e); err != nil { // the program's last line of standard output (A-254) + said = node + ": " + err.Error() + } else if verdict, said = runVerdict(node, answer); said == "" { + outcome = meshwants.CompositionRunOutcomePassed + } + _, err = r.endRun(key, e.Ask, outcome, said, verdict) + return err +} + +// await waits until each run of keys has ended, through the state's changes heard, or ctx ends: it answers each run as it +// stands then; a key gone could not run. +func (r *compositionRuns) await(ctx context.Context, keys []string) map[string]meshwants.CompositionRun { + out := map[string]meshwants.CompositionRun{} + for { + wake := r.wake() + open := false + for _, key := range keys { + run, _, ok, err := r.read(key) + switch { + case err != nil: + open = true + continue + case !ok: + at := r.now().UTC() + candidate, machine, _ := strings.Cut(key, ".") + run = meshwants.CompositionRun{Candidate: unescape(candidate), Machine: unescape(machine), Ended: &at, + Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: unescape(machine) + ": its composition run " + key + " is gone"} + case run.Ended == nil: + open = true + } + out[key] = run + } + if !open { + return out + } + select { + case <-ctx.Done(): + return out + case <-wake: + } + } +} + +// unescape is a key's part as a name again; a part that does not read stays as it is. +func unescape(part string) string { + if name, err := keys.UnescapeName(part); err == nil { + return name + } + return part +} + +// sharedOf names per machine the runs of keys and their asks (§729): from runs where given, else as each key stands. +func (r *compositionRuns) sharedOf(keys []string, runs map[string]meshwants.CompositionRun) []meshwants.Shared { + out := []meshwants.Shared{} + for _, key := range keys { + run, ok := runs[key] + if !ok { + if run, _, ok, _ = r.read(key); !ok { + continue + } + } + out = append(out, meshwants.Shared{Machine: run.Machine, Run: key, Ask: run.Ask}) + } + return out +} + +// leave removes the composition steps of a superseded or cancelled pipeline from every run's readers (§723, A-469n), and answers +// their asks' ids, each told to stop before a run is ended; a running run left with no reader has its ask dropped and ends could not run, "no reader is left" (A-469z), +// so a later step of the candidate starts a new run. A run other readers read goes on for them. +func (r *compositionRuns) leave(pipeline string, stop func(ids []string)) ([]string, error) { + all, err := r.keys() + if err != nil { + return nil, err + } + var ids []string + var errs []error + for _, key := range all { + if !isRunKey(key) { + continue + } + var left []string + var leaving []meshwants.CompositionReader + run, ok, err := r.change(key, func(c *meshwants.CompositionRun) bool { + left, leaving = nil, nil + keep := c.Readers[:0:0] + for _, reader := range c.Readers { + if reader.Pipeline == pipeline && reader.Step == meshwants.CompositionReaderStepComposition { + left = append(left, reader.ID) + leaving = append(leaving, reader) + continue + } + keep = append(keep, reader) + } + c.Readers = keep + return len(left) > 0 + }) + if err != nil { + errs = append(errs, err) + continue + } + if !ok || len(left) == 0 { + continue + } + ids = append(ids, left...) + if run.Ended == nil && r.untaken(key, run.Ask) { // a reader leaving a run not yet taken: its wait ends at once (A-470u) + r.sayWaits(run, leaving, "superseded") + } + if stop != nil { + stop(left) // the steps that left say no end, the run's included + } + if run.Ended != nil || len(run.Readers) > 0 { + continue + } + why := "no reader is left: the pipeline " + pipeline + " is superseded or cancelled" + if r.drop != nil { + if err := r.drop(run.Ask, why); err != nil { + r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err) + } + } + if _, err := r.endRun(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, run.Machine+": "+why, nil); err != nil { + errs = append(errs, err) + } + } + return ids, errors.Join(errs...) +} + +// passedOn is whether the step of module passed on the machine of an ended run (§722): neither failed nor could not run there. +func passedOn(module string, run meshwants.CompositionRun) bool { + e := judgeRun(module, run) + return e.failed == "" && e.couldNot == "" +} + +// formerOf is the place in readers of a step's former ask: the reader of the same pipeline's composition step, or -1. +func formerOf(readers []meshwants.CompositionReader, pipeline string) int { + return slices.IndexFunc(readers, func(c meshwants.CompositionReader) bool { + return c.Pipeline == pipeline && c.Step == meshwants.CompositionReaderStepComposition + }) +} + +// rejoin is join for a step asked again (§727, A-468g, A-470d): on a machine whose ended run it passed it keeps that verdict, +// reading the run under its new ask; a running run it joins; an ended run it did not read and that could run it reads, as one +// started after its last end; otherwise it starts a new run, replacing the ended key by compare-and-set and carrying every former +// reader into the new value: each that did not pass on the former run marked `rerun` with by the rerun's caller, each that passed +// not (A-470d). +func (r *compositionRuns) rejoin(reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) { + key := runKeyOf(candidate, machine) + for { + run, rev, ok, err := r.read(key) + if err != nil { + return joined{}, err + } + if !ok { + return r.join(reader, candidate, machine, ask) + } + at := formerOf(run.Readers, reader.Pipeline) + ended := run.Ended != nil + keep := ended && at >= 0 && passedOn(reader.Module, run) + read := !ended || (at < 0 && run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun) + if keep || read { + if at >= 0 && run.Readers[at].ID == reader.ID { + return joined{key: key, run: run}, nil + } + var former []meshwants.CompositionReader + if at >= 0 { + former = []meshwants.CompositionReader{run.Readers[at]} + run.Readers[at] = reader + } else { + run.Readers = append(run.Readers, reader) + } + if err := r.write(key, run, rev); errors.Is(err, errRunChanged) { + continue + } else if err != nil { + return joined{}, err + } + if run.Ended == nil && r.untaken(key, run.Ask) { // as join: a reader of a run not yet taken waits for a build node (A-469y) + r.sayWaits(run, former, "superseded") // the place it takes: that wait ends at once (A-470u) + r.sayWaits(run, []meshwants.CompositionReader{reader}, "") + } + return joined{key: key, run: run}, nil + } + if err := r.room(); err != nil { + return joined{}, err + } + started := r.now().UTC() + fresh := meshwants.CompositionRun{Candidate: candidate, Machine: machine, Ask: ask, Started: &started} + for i, c := range run.Readers { + if i == at { + continue + } + c.Rerun, c.By = !passedOn(c.Module, run), "" + if c.Rerun { + c.By = reader.By + } + fresh.Readers = append(fresh.Readers, c) + } + fresh.Readers = append(fresh.Readers, reader) + if err := r.clearBounds(key, run.Ask, runBoundPrefix, runUntakenPrefix); err != nil { // the former run's bounds go with it + r.logf("the bound keys of the former composition run %s are not deleted: %v", key, err) + } + if err := r.write(key, fresh, rev); errors.Is(err, errRunChanged) { + continue + } else if err != nil { + return joined{}, err + } + if _, err := r.armBound(runUntakenPrefix, key, ask, r.set.untaken); err != nil { + r.logf("the untaken bound of the composition run %s is not armed: %v", key, err) + } + r.sayWaits(fresh, fresh.Readers, "") + return joined{key: key, started: true, run: fresh}, nil + } +} + +// write writes a run's value at its revision. +func (r *compositionRuns) write(key string, run meshwants.CompositionRun, rev uint64) error { + raw, err := json.Marshal(run) + if err != nil { + return err + } + if _, err := r.kv.update(key, raw, rev); err != nil { + return err + } + r.heard() + return nil +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go new file mode 100644 index 00000000..7d32c033 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go @@ -0,0 +1,404 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" + + "mesh-delivery-assigner/internal/buckets" +) + +// Test 108 (design 47 G §733; §718 to §723, §727, §728; A-468 to A-470f): the assigner started whole against the bus composes +// each machine once per candidate. Two or three composition steps of one candidate on two machines make one key, one request and +// one run ask per machine, and each step reads both verdicts; a machine blaming a and b fails their steps and passes c's with a +// warning; a verdict with no `blamed` is judged as before, "the composer names no culprits", and a machine blaming none fails +// every step; a step of another candidate starts its own runs. A taken ask that keeps saying progress is not ended by the bound, +// one silent past it ends could not run with its ask dropped, a step-ended after that is ignored and journaled, and an ask never +// taken ends at run-untaken-bound; a bound holds its run's ask (A-470n). A rerun of a starts one run on the machine it did not +// pass and moves b; a superseded step leaves +// a run running for its other reader; an assigner restarted during a run takes it up by its ask; compositions-drop ends the +// readers and deletes the key; at the cap a step ends could not run, naming the fill. +func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) { + b := startBed(t, "") + b.settings = map[string]string{"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "2s", "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "15s", // past a restart + "MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "1"} + nc, err := nats.Connect(b.srv.ClientURL()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + const commit = "1111111111111111111111111111111111111111" + if _, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner", + FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll, + nodebuildgo.SubjectStepEndedAll, nodebuildgo.SubjectAskTakenAll, nodebuildgo.SubjectStepProgressAll, pipelineEndedSubject}, + AckPolicy: jetstream.AckExplicitPolicy, AckWait: 2 * time.Second}); err != nil { // an event handed to the stopped assigner's pull comes again soon + t.Fatal(err) + } + for _, node := range []string{"n1", "n2"} { + raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: node, Module: "engine", Fingerprint: "src2:e1", At: time.Now().UTC()}) + if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, "engine"), raw); err != nil { + t.Fatal(err) + } + } + awaitKept(t, b, nodeengine.StreamRunning, 2) + engine := meshcatalog.CatalogEntry{Module: "engine", Fp: "src2:e1", Repository: "novox/mesh-host", Commit: commit, Path: ".", + Serves: []string{nodeengine.Seat + ".composition"}, Manifest: map[string]any{"module": "engine", "version": "0.1"}} + b.cat.mu.Lock() + b.cat.entries[trunkBranch] = map[string]meshcatalog.CatalogEntry{"engine": engine} + for _, m := range []string{"a", "b", "c"} { + b.cat.entries[trunkBranch][m] = meshcatalog.CatalogEntry{Module: m, Fp: "src2:" + m, Repository: "novox/" + m, Commit: commit, Path: ".", Manifest: map[string]any{"module": m}} + } + b.cat.rev[trunkBranch] = 3 + b.cat.mu.Unlock() + for _, node := range []string{"n1", "n2"} { + for _, m := range []string{"a", "b", "c"} { + b.place(t, node, m) + } + } + var mu sync.Mutex + inputsAsked := map[string]int{} // by candidate and node + var drops []string + respond := func(subject string, f func(m *nats.Msg) any) { + sub, err := nc.Subscribe(subject, func(m *nats.Msg) { + raw, _ := json.Marshal(map[string]any{"result": f(m), "node": "n0"}) + _ = m.Respond(raw) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + } + respond("mesh.seat.node-engine.call.compose-candidate.*.*", func(m *nats.Msg) any { + node := strings.Split(m.Subject, ".")[5] + var in nodeengine.ComposeCandidateInput + _ = json.Unmarshal(m.Data, &in) + mu.Lock() + inputsAsked[in.Candidate+" "+node]++ + mu.Unlock() + return map[string]any{"candidate": in.Candidate, "node": node, "verdict": "composed", "input": map[string]any{"node": node}, "modules": []any{}, "own": map[string]any{}, "took": 1, "v": 1} + }) + respond("mesh.seat.mesh-catalog.call.entry.*", func(*nats.Msg) any { return meshcatalog.EntryAnswer{Entry: engine} }) + respond("mesh.seat.node-build-go.call.drop.*", func(m *nats.Msg) any { + var in nodebuildgo.DropInput + _ = json.Unmarshal(m.Data, &in) + mu.Lock() + drops = append(drops, in.ID) + mu.Unlock() + return map[string]any{} + }) + respond("mesh.seat.mesh-hand-acts.call.hand-act.*", func(*nats.Msg) any { return map[string]any{"log": "hand-act-1"} }) + // The ends are read from the stream through a durable consumer, as the runner hears them: an end the bus drops as a + // repeated message id is never heard (A-470q). + ended := make(chan meshwants.StepEnded, 32) + runner, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "runner-stand-in", + FilterSubject: "mesh.seat.mesh-wants.event.step-ended.n1", AckPolicy: jetstream.AckExplicitPolicy, DeliverPolicy: jetstream.DeliverNewPolicy}) + if err != nil { + t.Fatal(err) + } + heard, err := runner.Consume(func(m jetstream.Msg) { + var e meshwants.StepEnded + _ = json.Unmarshal(m.Data(), &e) + _ = m.Ack() + ended <- e + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(heard.Stop) + subscribed(t, nc) + + from := b.journal.mark() + _, stop := b.startStoppable(t) + b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second) + select { + case <-b.seatListRead: + case <-time.After(30 * time.Second): + t.Fatal("the assigner did not read the seat list within 30 s") + } + queue, _ := buildstep.AskStream(buildstep.AskGo) + worker, err := b.js.CreateOrUpdateConsumer(b.ctx, queue, jetstream.ConsumerConfig{Durable: "stand-in", FilterSubject: "mesh.seat.node-build-go.accept.check.mesh-wants", + AckPolicy: jetstream.AckExplicitPolicy}) + if err != nil { + t.Fatal(err) + } + step := func(id, module, run string, rerun bool) { + t.Helper() + s := meshwants.StepRun{Pipeline: "main.3." + module, Step: meshwants.StepRunStepComposition, Module: module, Branch: trunkBranch, Revision: 3, + Commit: commit, Fingerprint: "src2:" + module, Run: run, Rerun: rerun} + if run != "" { + s.Pipeline = "run-" + run + "." + module + } + if rerun { + s.By = "op@n1" + } + raw, _ := json.Marshal(s) + msg := nats.NewMsg(meshwants.AcceptSubjectStep) + msg.Header.Set("x-event-id", id) + msg.Data = raw + if err := nc.PublishMsg(msg); err != nil { + t.Fatal(err) + } + } + type asked struct { + ask buildstep.StepAsk + machine string + } + // asks takes n run asks of the stand-in build seat, each with the machine its request composes. + asks := func(n int) []asked { + t.Helper() + var out []asked + for range n { + got, err := worker.Next(jetstream.FetchMaxWait(20 * time.Second)) + if err != nil { + t.Fatalf("%d of %d run asks put: %v", len(out), n, err) + } + _ = got.Ack() + a, err := buildstep.DecodeStepAsk(got.Data()) + if err != nil || a.Run == nil { + t.Fatalf("the run ask %s does not read: %v", got.Data(), err) + } + req, err := composer.Decode(readInput(t, nc, a.Run.Input)) + if err != nil || len(req.Machines) != 1 || req.BlameTries != 20 { + t.Fatalf("the request of %s: %+v %v", a.ID, req, err) + } + out = append(out, asked{a, req.Machines[0]}) + } + return out + } + noMoreAsks := func(what string) { + t.Helper() + if got, err := worker.Next(jetstream.FetchMaxWait(300 * time.Millisecond)); err == nil { + t.Fatalf("%s: one more run ask is put: %s", what, got.Data()) + } + } + say := func(event string, node string, body any) { + t.Helper() + raw, _ := json.Marshal(body) + if _, err := b.js.Publish(b.ctx, "mesh.seat.node-build-go.event."+event+"."+node, raw); err != nil { + t.Fatal(err) + } + } + answer := func(a asked, machine string) { + t.Helper() + v := fmt.Sprintf(`{"v":%d,"cases":[{"name":"change","manifests":{},"machines":{%q:%s}}]}`, composer.Version, a.machine, machine) + say("step-ended", "b1", buildstep.StepEnded{V: 1, ID: a.ask.ID, Ask: a.ask.ID, Asker: runAsker, Node: "b1", Outcome: buildstep.OutcomePassed, Output: v, At: time.Now().UTC()}) + } + ends := func(n int, what string) map[string]meshwants.StepEnded { + t.Helper() + out := map[string]meshwants.StepEnded{} + for len(out) < n { + select { + case e := <-ended: + out[e.ID] = e + case <-time.After(20 * time.Second): + t.Fatalf("%s: %d of %d steps ended: %+v", what, len(out), n, out) + } + } + return out + } + runKeys := func(candidate string) []string { + all, _ := b.buckets[buckets.CompositionRuns()].Keys(b.ctx) + var out []string + for _, k := range all { + if isRunKey(k) && strings.HasPrefix(k, keysPart(candidate)+".") { + out = append(out, k) + } + } + return out + } + // readers waits, through a watch of the key, until the run of key has n readers. + readers := func(key string, n int) { + t.Helper() + w, err := b.buckets[buckets.CompositionRuns()].Watch(b.ctx, key) + if err != nil { + t.Fatal(err) + } + defer func() { _ = w.Stop() }() + deadline := time.After(10 * time.Second) + for { + select { + case e := <-w.Updates(): + var run meshwants.CompositionRun + if e != nil && json.Unmarshal(e.Value(), &run) == nil && len(run.Readers) == n { + return + } + case <-deadline: + t.Fatalf("the run %s has not %d readers", key, n) + } + } + } + blames := `{"composes":false,"reason":"two modules own one path","blamed":[{"module":"a","problem":"/etc/x is a's and b's"},{"module":"b","problem":"/etc/x is a's and b's"}]}` + composes := `{"composes":true,"blamed":[]}` + + // One candidate, three steps, two machines: two runs. + for _, m := range []string{"a", "b", "c"} { + step("step-"+m, m, "", false) + } + first := asks(2) + noMoreAsks("three steps of one candidate") + for _, a := range first { + answer(a, map[string]string{"n1": composes, "n2": blames}[a.machine]) + } + got := ends(3, "the three steps") + if a, bb, c := got["step-a"], got["step-b"], got["step-c"]; a.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(a.Said, "n2 holds a") || + bb.Outcome != meshwants.StepEndedOutcomeFailed || c.Outcome != meshwants.StepEndedOutcomePassed || len(c.Warnings) != 2 || + !strings.Contains(c.Warnings[0], "n2 holds a") || len(c.Shared) != 2 || c.Shared[0].Ask == c.Shared[1].Ask { + t.Fatalf("the steps of one candidate: %+v", got) + } + if keys := runKeys("main.3"); len(keys) != 2 || inputsAsked["main.3 n1"] != 1 || inputsAsked["main.3 n2"] != 1 { + t.Fatalf("keys %v, inputs asked %v: want one run and one input per machine", keys, inputsAsked) + } + + // Another candidate (a run on request): its own runs; an old composer and a machine blaming none fail. + step("step-r1", "a", "r1", false) + for _, a := range asks(2) { + answer(a, map[string]string{"n1": `{"composes":false,"reason":"x"}`, "n2": `{"composes":false,"reason":"own","blamed":[]}`}[a.machine]) + } + if e := ends(1, "another candidate")["step-r1"]; e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose and blames no module") || + len(e.Warnings) != 1 || e.Warnings[0] != "n1: the composer names no culprits" { + t.Fatalf("a step of another candidate: %+v", e) + } + + // The bound: n1's ask taken and saying progress past the bound, then silent; n2's never taken. + from = b.journal.mark() + step("step-r2", "a", "r2", false) + bound := asks(2) + var n1 asked + for _, a := range bound { + if a.machine == "n1" { + n1 = a + } + } + say("ask-taken", "b1", nodebuildgo.AskTaken{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Node: "b1"}) + for i := range 3 { // past the 2 s bound, each beat within it + select { + case e := <-ended: + t.Fatalf("a run saying its progress ended: %+v", e) + case <-time.After(time.Second): + } + if i == 0 { // the bound holds its run's ask, published at the key's latest revision with its own time to live (A-470n) + runs := b.buckets[buckets.CompositionRuns()] + all, _ := runs.Keys(b.ctx) + held := "" + for _, k := range all { + if strings.HasPrefix(k, runBoundPrefix) { + if e, err := runs.Get(b.ctx, k); err == nil { + var v boundValue + _ = json.Unmarshal(e.Value(), &v) + held += v.Ask + } + } + } + if held != n1.ask.ID { + t.Fatalf("the bound keys hold the asks %q, want n1's %s alone", held, n1.ask.ID) + } + } + say("step-progress", "b1", nodebuildgo.StepProgress{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Phase: "run"}) + } + e := ends(1, "the bounds")["step-r2"] + if e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "said no progress within 2s (composition-step-bound)") || + !strings.Contains(e.Said, "taken by no build node within 15s (run-untaken-bound)") { + t.Fatalf("past the bounds: %+v", e) + } + mu.Lock() + dropped := len(drops) + mu.Unlock() + if dropped != 2 { + t.Fatalf("%d asks dropped past their bounds, want 2", dropped) + } + answer(n1, composes) // heard after the bound: ignored and journaled + b.journal.await(t, from, "the step-ended of the run ask "+n1.ask.ID+" is ignored", "a late step-ended", 10*time.Second) + + // A rerun of a: n1 it passed keeps its verdict; n2 it failed starts one run carrying b, which moves. + step("step-a2", "a", "", true) + again := asks(1) + noMoreAsks("a rerun of a") + if again[0].machine != "n2" { + t.Fatalf("the rerun composes %s, want only the machine a did not pass", again[0].machine) + } + answer(again[0], composes) + moved := ends(2, "the rerun") + if a2, bb := moved["step-a2"], moved["step-b"]; a2.Outcome != meshwants.StepEndedOutcomePassed || bb.Outcome != meshwants.StepEndedOutcomePassed || + !bb.Rerun || bb.By != "op@n1" || len(bb.Shared) != 2 || bb.Shared[1].Ask != again[0].ask.ID { + t.Fatalf("after the rerun (asked again %s): a2 %+v; b %+v", again[0].ask.ID, moved["step-a2"].Shared, moved["step-b"]) + } + + // A superseded step leaves the runs; its other reader goes on. + step("step-r3a", "a", "r3", false) + step("step-r3b", "b", "r3", false) + sup := asks(2) + readers(runKeyOf("run-r3", "n1"), 2) + readers(runKeyOf("run-r3", "n2"), 2) + pe, _ := json.Marshal(mpr.PipelineEnded{V: 1, Pipeline: "run-r3.a", Module: "a", Branch: trunkBranch, Commit: commit, Outcome: mpr.PipelineEndedOutcomeSuperseded, Run: "r3", Steps: []mpr.StepResult{}}) + if _, err := b.js.Publish(b.ctx, "mesh.seat."+mpr.Seat+".event."+mpr.EventPipelineEnded+".n1", pe); err != nil { + t.Fatal(err) + } + for _, a := range sup { + answer(a, composes) + } + if got := ends(1, "the superseded run"); got["step-r3b"].Outcome != meshwants.StepEndedOutcomePassed { + t.Fatalf("the other reader: %+v", got) + } + select { + case e := <-ended: + t.Fatalf("a superseded step said its end: %+v", e) + case <-time.After(200 * time.Millisecond): + } + + // Restarted during a run: the new assigner takes it up by its ask. + step("step-r4", "c", "r4", false) + restart := asks(2) + stop() + for _, a := range restart { + answer(a, composes) + } + from = b.journal.mark() + b.startStoppable(t) + b.journal.await(t, from, "is taken up again by its runs (ask step-r4)", "the take-up", 30*time.Second) + if e := ends(1, "the take-up")["step-r4"]; e.Outcome != meshwants.StepEndedOutcomePassed || len(e.Shared) != 2 { + t.Fatalf("the step taken up: %+v", e) + } + + // compositions-drop: the readers ended naming the drop, the key deleted. + step("step-r5", "a", "r5", false) + asks(2) + body, _ := json.Marshal(meshwants.CompositionsDropInput{Candidate: "run-r5", Machine: "n1", Why: "stuck"}) + reply, err := nc.Request("mesh.seat.mesh-wants.call.compositions-drop.n1~mesh-delivery-pipeline", body, 10*time.Second) + if err != nil || !strings.Contains(string(reply.Data), `"ended":[`) { + t.Fatalf("compositions-drop answered %s %v", reply.Data, err) + } + if e := ends(1, "the drop")["step-r5"]; e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "dropped by hand: stuck") { + t.Fatalf("a reader of the dropped run: %+v", e) + } + if keys := runKeys("run-r5"); len(keys) != 1 { + t.Errorf("the keys of run-r5 after the drop: %v, want n2's alone", keys) + } + + // At the cap of 1 MiB no run starts: the step ends could not run, naming the fill. + for i := range 2 { + if _, err := b.buckets[buckets.CompositionRuns()].Put(b.ctx, fmt.Sprintf("fill%d", i), make([]byte, 700<<10)); err != nil { + t.Fatal(err) + } + } + step("step-r6", "a", "r6", false) + if e := ends(1, "the cap")["step-r6"]; e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "is at its cap of 1 MiB") { + t.Fatalf("at the cap: %+v", e) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go new file mode 100644 index 00000000..7a0ccf15 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go @@ -0,0 +1,642 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "time" + + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" +) + +// runsMem is the state composition-runs in memory, its bound keys' times to live kept beside. +type runsMem struct { + mu sync.Mutex + vals map[string][]byte + revs map[string]uint64 + ttl map[string]time.Duration + bytes int64 + n uint64 + // beforeDel, when set, runs once before the next delete: another write landing between a read and the delete. + beforeDel func() + // beforeArm, when set, runs once before the next arm: another write landing between a read and the arm. + beforeArm func() +} + +func (k *runsMem) another() { + k.mu.Lock() + f := k.beforeDel + k.beforeDel = nil + k.mu.Unlock() + if f != nil { + f() + } +} + +func newRunsMem() *runsMem { + return &runsMem{vals: map[string][]byte{}, revs: map[string]uint64{}, ttl: map[string]time.Duration{}} +} + +func (k *runsMem) get(key string) ([]byte, uint64, error) { + k.mu.Lock() + defer k.mu.Unlock() + return k.vals[key], k.revs[key], nil +} + +func (k *runsMem) create(key string, v []byte) (uint64, error) { + k.mu.Lock() + defer k.mu.Unlock() + if _, ok := k.vals[key]; ok { + return 0, errRunChanged + } + k.n++ + k.vals[key], k.revs[key] = v, k.n + return k.n, nil +} + +func (k *runsMem) update(key string, v []byte, rev uint64) (uint64, error) { + k.mu.Lock() + defer k.mu.Unlock() + if k.revs[key] != rev { + return 0, errRunChanged + } + k.n++ + k.vals[key], k.revs[key] = v, k.n + return k.n, nil +} + +func (k *runsMem) delAt(key string, rev uint64) error { + k.another() + k.mu.Lock() + defer k.mu.Unlock() + if _, ok := k.vals[key]; !ok { + return nil + } + if k.revs[key] != rev { + return errRunChanged + } + delete(k.vals, key) + delete(k.revs, key) + delete(k.ttl, key) + return nil +} + +func (k *runsMem) keys() ([]string, error) { + k.mu.Lock() + defer k.mu.Unlock() + var out []string + for key := range k.vals { + out = append(out, key) + } + slices.Sort(out) + return out, nil +} + +func (k *runsMem) arm(key, ask string, ttl time.Duration) (uint64, error) { + k.mu.Lock() + f := k.beforeArm + k.beforeArm = nil + k.mu.Unlock() + if f != nil { + f() + } + raw, _ := json.Marshal(boundValue{Bound: key, Ask: ask}) + k.mu.Lock() + defer k.mu.Unlock() + k.n++ + k.vals[key], k.revs[key], k.ttl[key] = raw, k.n, ttl + return k.n, nil +} + +// expire ends a bound key's time to live: the key gone, as the bus's marker leaves it. +func (k *runsMem) expire(key string) { + k.mu.Lock() + defer k.mu.Unlock() + delete(k.vals, key) + delete(k.revs, key) + delete(k.ttl, key) +} + +func (k *runsMem) used() (int64, error) { + k.mu.Lock() + defer k.mu.Unlock() + if k.bytes > 0 { + return k.bytes, nil + } + var n int64 + for _, v := range k.vals { + n += int64(len(v)) + } + return n, nil +} + +// Design 47 G §718, §719, §720, A-468, A-469d, A-469x: a run's key is `.`, each part escaped; two +// steps of one candidate joining at once start one run and the second reads it; a run that could not run is replaced; at the +// cap no run starts, naming the fill; a run is deleted composition-runs-kept after its end; the settings are read from their files. +func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) { + now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC) + step := func(id, module string) meshwants.CompositionReader { + return readerOf(id, meshwants.StepRun{Pipeline: "main.3." + module, Step: meshwants.StepRunStepComposition, Module: module, Branch: "main", Revision: 3}) + } + t.Run("key", func(t *testing.T) { + if k := runKeyOf("main.3", "n1"); k != "main=2e3.n1" { + t.Errorf("the key is %q, want each part escaped as a key's name", k) + } + if isRunKey(runBoundPrefix+"main=2e3.n1") || isRunKey(runUntakenPrefix+"main=2e3.n1") || !isRunKey("main=2e3.n1") { + t.Error("a bound key is taken for a run's or a run's for a bound key") + } + }) + t.Run("one run for two steps at once", func(t *testing.T) { + kv := newRunsMem() + r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf) + var wg sync.WaitGroup + got := make([]joined, 2) + for i, m := range []string{"a", "b"} { + wg.Add(1) + go func() { + defer wg.Done() + j, err := r.join(step("ask-"+m, m), "main.3", "n1", "run-"+m) + if err != nil { + t.Error(err) + } + got[i] = j + }() + } + wg.Wait() + runKeys := func() []string { + all, _ := kv.keys() + return slices.DeleteFunc(all, func(k string) bool { return !isRunKey(k) }) + } + if got[0].started == got[1].started { + t.Fatalf("started %v and %v: want exactly one step to start the run", got[0].started, got[1].started) + } + keys := runKeys() + run, _, _, _ := r.read(runKeyOf("main.3", "n1")) + if len(keys) != 1 || len(run.Readers) != 2 || run.Candidate != "main.3" || run.Machine != "n1" || run.Started == nil { + t.Errorf("keys %v, run %+v: want one key read by both", keys, run) + } + if j, _ := r.join(step("ask-a", "a"), "main.3", "n1", "run-c"); j.started || len(j.run.Readers) != 2 { + t.Errorf("a reader joining again is added again or starts a run: %+v", j) + } + }) + t.Run("could not run is never read", func(t *testing.T) { + kv := newRunsMem() + r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf) + if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil { + t.Fatal(err) + } + key := runKeyOf("main.3", "n1") + _, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool { + c.Ended, c.Outcome = &now, meshwants.CompositionRunOutcomeCouldNotRun + return true + }) + j, err := r.join(step("ask-b", "b"), "main.3", "n1", "run-2") + if err != nil || !j.started || j.run.Ask != "run-2" || len(j.run.Readers) != 1 || j.run.Ended != nil { + t.Errorf("after a could not run: %+v %v, want a new run read by its starter alone", j, err) + } + _, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool { + c.Ended, c.Outcome = &now, meshwants.CompositionRunOutcomePassed + return true + }) + if j, _ := r.join(step("ask-c", "c"), "main.3", "n1", "run-3"); j.started || j.run.Ask != "run-2" { + t.Errorf("a passed run is not read: %+v", j) + } + if j, _ := r.join(step("ask-d", "d"), "main.4", "n1", "run-4"); !j.started { + t.Errorf("a step of another candidate reads the first's run: %+v", j) + } + }) + t.Run("cap", func(t *testing.T) { + kv := newRunsMem() + kv.bytes = 16 << 20 + r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf) + _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1") + if err == nil || !strings.Contains(err.Error(), "at its cap of 16 MiB (16777216 bytes kept)") { + t.Errorf("at the cap: %v, want the fill named", err) + } + }) + t.Run("kept", func(t *testing.T) { + kv := newRunsMem() + at := now + r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return at }, t.Logf) + for _, c := range []string{"main.1", "main.2"} { + if _, err := r.join(step("ask-"+c, "a"), c, "n1", "run-"+c); err != nil { + t.Fatal(err) + } + } + old := now.Add(-25 * time.Hour) + _, _, _ = r.change(runKeyOf("main.1", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &old, "passed"; return true }) + _, _ = kv.arm(runBoundPrefix+runKeyOf("main.1", "n1"), "run-main.1", time.Minute) + recent := now.Add(-time.Hour) + _, _, _ = r.change(runKeyOf("main.2", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &recent, "passed"; return true }) + if _, err := r.join(step("ask-x", "a"), "main.9", "n1", "run-9"); err != nil { + t.Fatal(err) + } + keys, _ := kv.keys() + for _, k := range []string{runBoundPrefix, runUntakenPrefix} { + if v, _, _ := kv.get(k + runKeyOf("main.1", "n1")); v != nil { + t.Errorf("the key %s stays past its run", k+runKeyOf("main.1", "n1")) + } + } + keys = slices.DeleteFunc(keys, func(k string) bool { return !isRunKey(k) }) + if want := []string{runKeyOf("main.2", "n1"), runKeyOf("main.9", "n1")}; !slices.Equal(keys, want) { + t.Errorf("keys %v, want %v: a run ended more than a day ago goes with its bound key at the next run", keys, want) + } + }) + t.Run("a listing that misses a key being written", func(t *testing.T) { + kv := &missingRuns{runsMem: newRunsMem()} + r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf) + if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil { + t.Fatal(err) + } + kv.hide = runKeyOf("main.3", "n1") + if key, _, _ := r.keyOfAsk("run-1"); key != "" { + t.Fatalf("the stand's listing does not hide the key") + } + r.saw(kv.hide, true) // as the state's watch hands it + if key, _, _ := r.keyOfAsk("run-1"); key != kv.hide { + t.Errorf("the run of ask run-1 is not found by the keys the watch handed: %q", key) + } + r.saw(kv.hide, false) + if keys, _ := r.keys(); slices.Contains(keys, kv.hide) { + t.Errorf("a key the watch saw deleted is still listed: %v", keys) + } + }) + // No delete lands on a value newer than the one it was judged from (the kind test 109 met in test-runs): a rerun's run + // replacing the key, and its untaken bound, stand. + for _, c := range []struct { + name string + ended time.Time + act func(r *compositionRuns, key string) error + }{ + {"the sweep of a run replaced meanwhile", now.Add(-25 * time.Hour), func(r *compositionRuns, _ string) error { return r.sweep() }}, + {"the end of a run replaced meanwhile", time.Time{}, func(r *compositionRuns, key string) error { + _, err := r.endRun(key, "run-1", meshwants.CompositionRunOutcomePassed, "", nil) + return err + }}, + } { + t.Run(c.name, func(t *testing.T) { + kv := newRunsMem() + r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf) + if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil { + t.Fatal(err) + } + key := runKeyOf("main.3", "n1") + if !c.ended.IsZero() { + _, _, _ = r.change(key, func(run *meshwants.CompositionRun) bool { run.Ended, run.Outcome = &c.ended, "failed"; return true }) + } + _, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute) + kv.beforeDel = func() { // a rerun replaces the key and arms its untaken bound + for { + raw, rev, _ := kv.get(key) + var run meshwants.CompositionRun + _ = json.Unmarshal(raw, &run) + if run.Ended == nil { + at := now + run.Ended, run.Outcome = &at, "failed" + } + fresh, _ := json.Marshal(meshwants.CompositionRun{Candidate: "main.3", Machine: "n1", Ask: "run-2", Started: &now, Readers: run.Readers}) + if _, err := kv.update(key, fresh, rev); err == nil { + break + } + } + _, _ = kv.arm(runUntakenPrefix+key, "run-2", time.Hour) + } + if err := c.act(r, key); err != nil { + t.Fatal(err) + } + if run, _, ok, _ := r.read(key); !ok || run.Ask != "run-2" || run.Ended != nil { + t.Errorf("the key holds %+v (stands %v), want the rerun's running run-2", run, ok) + } + if v, _, _ := kv.get(runUntakenPrefix + key); v == nil { + t.Error("the untaken bound the rerun's run armed is deleted") + } + }) + } + t.Run("settings", func(t *testing.T) { + dir := t.TempDir() + files := map[string]string{"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "8", "MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": "2h", + "MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "2s", "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "", "MESH_ASSIGNER_BLAME_TRIES_FILE": "5"} + env := map[string]string{} + for name, content := range files { + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + env[name] = p + } + s, err := runSettingsOf(func(k string) string { return env[k] }) + if err != nil || s.capBytes != 8<<20 || s.kept != 2*time.Hour || s.stepBound != 2*time.Second || s.untaken != 24*time.Hour || s.blameTries != 5 { + t.Errorf("settings %+v %v", s, err) + } + _ = os.WriteFile(env["MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE"], []byte("30"), 0o600) + if _, err := runSettingsOf(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "not a positive Go duration") { + t.Errorf("a bound of no unit: %v", err) + } + if s, err := runSettingsOf(func(string) string { return "" }); err != nil || s != defaultRunSettings { + t.Errorf("no files: %+v %v, want the defaults", s, err) + } + }) + raw, _ := json.Marshal(step("x", "a")) + if !strings.Contains(string(raw), `"id":"x"`) { + t.Errorf("a reader names no ask id: %s", raw) + } +} + +// Design 47 G §720, A-466s, A-468b, A-469y, A-470: a run's untaken bound is armed at its start and each reader said waiting for a +// build node; ask-taken arms the bound of composition-step-bound and ends the waits ran; each step-progress arms it again; the +// bound's expiry drops the ask and ends the run could not run naming the ask and the bound, a step-ended heard later ignored and +// journaled; the untaken bound's expiry ends it naming run-untaken-bound, the waits superseded; the readers of a run with no end +// are taken up at the start. +func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) { + now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC) + reader := func(id string) meshwants.CompositionReader { + return readerOf(id, meshwants.StepRun{Pipeline: "main.3." + id, Step: meshwants.StepRunStepComposition, Module: id, Branch: "main", Revision: 3}) + } + type said struct{ event, how, module string } + setup := func(t *testing.T) (*runsMem, *compositionRuns, *[]said, *[]string, *[]string) { + kv := newRunsMem() + var saids []said + var drops, journal []string + var mu sync.Mutex + r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, func(f string, a ...any) { + mu.Lock() + journal = append(journal, fmt.Sprintf(f, a...)) + mu.Unlock() + }) + r.say = func(event string, body any, _ string) error { + w := body.(meshwants.Wait) + if w.On != waitOnBuildNode || w.Step != "pipeline" || w.Owner != meshwants.Seat || w.Node == "" { + t.Errorf("the wait said %+v", w) + } + saids = append(saids, said{event, w.How, w.Module}) + return nil + } + r.drop = func(ask, why string) error { drops = append(drops, ask+": "+why); return nil } + return kv, r, &saids, &drops, &journal + } + key := runKeyOf("main.3", "n1") + t.Run("taken, progress, bound", func(t *testing.T) { + kv, r, saids, drops, journal := setup(t) + if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil { + t.Fatal(err) + } + if _, err := r.join(reader("b"), "main.3", "n1", "run-2"); err != nil { + t.Fatal(err) + } + if kv.ttl[runUntakenPrefix+key] != 24*time.Hour || !slices.Equal(*saids, []said{{meshwants.EventWaitHeld, "", "a"}, {meshwants.EventWaitHeld, "", "b"}}) { + t.Fatalf("at the start: untaken %v, said %v", kv.ttl, *saids) + } + if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: "someone-else"}); err != nil || kv.ttl[runBoundPrefix+key] != 0 { + t.Fatalf("another asker's ask-taken arms a bound: %v %v", kv.ttl, err) + } + if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: runAsker, Node: "b1"}); err != nil { + t.Fatal(err) + } + _, untaken, _ := kv.get(runUntakenPrefix + key) + if kv.ttl[runBoundPrefix+key] != 30*time.Minute || untaken != 0 || len(*saids) != 4 || (*saids)[2].how != "ran" || (*saids)[3].how != "ran" { + t.Fatalf("at ask-taken: %v, said %v", kv.ttl, *saids) + } + before := kv.revs[runBoundPrefix+key] + if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil || kv.revs[runBoundPrefix+key] == before { + t.Fatalf("a step-progress does not arm the bound again: %v", err) + } + kv.expire(runBoundPrefix + key) + r.expired(runBoundPrefix + key) + run, _, _, _ := r.read(key) + if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 said no progress within 30m0s (composition-step-bound)" || + len(*drops) != 1 || !strings.HasPrefix((*drops)[0], "run-1: ") { + t.Fatalf("past the bound: %+v, drops %v", run, *drops) + } + if got := judgeRun("a", run); got.couldNot != run.Said { + t.Errorf("a reader of the run past its bound: %+v", got) + } + late := buildstep.StepEnded{Ask: "run-1", Asker: runAsker, Outcome: buildstep.OutcomePassed, Output: "{}"} + if err := r.heardEnd(late, func(e buildstep.StepEnded) ([]byte, error) { return []byte(e.Output), nil }); err != nil { + t.Fatal(err) + } + if again, _, _, _ := r.read(key); again.Said != run.Said || !slices.ContainsFunc(*journal, func(l string) bool { + return strings.Contains(l, "step-ended of the run ask run-1 is ignored") && strings.Contains(l, key) + }) { + t.Errorf("a step-ended after the bound: %+v, journal %v", again, *journal) + } + if keys, _ := kv.keys(); !slices.Equal(keys, []string{key}) { + t.Errorf("the bound keys stay past the run's end: %v", keys) + } + }) + t.Run("untaken", func(t *testing.T) { + kv, r, saids, drops, _ := setup(t) + if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil { + t.Fatal(err) + } + kv.expire(runUntakenPrefix + key) + r.expired(runUntakenPrefix + key) + run, _, _, _ := r.read(key) + if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 was taken by no build node within 24h0m0s (run-untaken-bound)" || + len(*drops) != 1 || !slices.Equal(*saids, []said{{meshwants.EventWaitHeld, "", "a"}, {meshwants.EventWaitEnded, "superseded", "a"}}) { + t.Errorf("untaken past its bound: %+v, drops %v, said %v", run, *drops, *saids) + } + r.expired(runUntakenPrefix + key) // a run ended takes nothing + if len(*drops) != 1 { + t.Errorf("an ended run is dropped again: %v", *drops) + } + }) + // A-470n, A-470o: a bound holds its run's ask, armed for it alone; a late signal of a run replaced moves nothing and is + // journaled; an expiry ends the run only while its bound is still absent and the run taken (bound.) or untaken (untaken.). + rerun := func(kv *runsMem, r *compositionRuns) func() { // a rerun replaces run-1 by run-2, its bounds gone, run-2's untaken armed + return func() { + _, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool { c.Ask = "run-2"; return true }) + kv.expire(runBoundPrefix + key) + _, _ = kv.arm(runUntakenPrefix+key, "run-2", time.Hour) + } + } + taken := func(t *testing.T, r *compositionRuns) { + if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: runAsker, Node: "b1"}); err != nil { + t.Fatal(err) + } + } + for _, c := range []struct { + name string + act func(t *testing.T, kv *runsMem, r *compositionRuns) + ended bool + bound string // the ask the bound key holds, "" for none + untaken string + journal string + }{ + {"a late ask-taken of a run replaced as it is heard", func(t *testing.T, kv *runsMem, r *compositionRuns) { + kv.beforeArm = rerun(kv, r) + taken(t, r) + }, false, "", "run-2", "the ask-taken of the run ask run-1 is late"}, + {"a late step-progress of a run replaced as it is heard", func(t *testing.T, kv *runsMem, r *compositionRuns) { + taken(t, r) + kv.beforeArm = rerun(kv, r) + if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil { + t.Fatal(err) + } + kv.expire(runBoundPrefix + key) // a bound left standing would end run-2 here + r.expired(runBoundPrefix + key) + }, false, "", "run-2", "the step-progress of the run ask run-1 is late"}, + {"a step-progress of a run gone", func(t *testing.T, kv *runsMem, r *compositionRuns) { + taken(t, r) + rerun(kv, r)() + if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil { + t.Fatal(err) + } + }, false, "", "run-2", "the step-progress of the run ask run-1 is late"}, + {"an expiry heard after the bound was armed again", func(t *testing.T, kv *runsMem, r *compositionRuns) { + taken(t, r) + r.expired(runBoundPrefix + key) + }, false, "run-1", "", "the expiry of " + runBoundPrefix + key + " ends nothing"}, + {"the bound's expiry of a run not taken", func(t *testing.T, kv *runsMem, r *compositionRuns) { + _, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute) + kv.expire(runBoundPrefix + key) + r.expired(runBoundPrefix + key) + }, false, "", "run-1", "the expiry of " + runBoundPrefix + key + " ends nothing"}, + {"the untaken expiry of a run taken", func(t *testing.T, kv *runsMem, r *compositionRuns) { + _, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute) + kv.expire(runUntakenPrefix + key) + r.expired(runUntakenPrefix + key) + }, false, "run-1", "", "the expiry of " + runUntakenPrefix + key + " ends nothing"}, + {"an end of another ask", func(t *testing.T, kv *runsMem, r *compositionRuns) { + if ended, err := r.end(key, "run-0", meshwants.CompositionRunOutcomePassed, "", nil); ended || err != nil { + t.Errorf("the end of run-0 ended the run of run-1: %v %v", ended, err) + } + }, false, "", "run-1", ""}, + {"the ask-taken of the run's own ask", func(t *testing.T, kv *runsMem, r *compositionRuns) { + taken(t, r) + }, false, "run-1", "", ""}, + } { + t.Run(c.name, func(t *testing.T) { + kv, r, _, _, journal := setup(t) + if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil { + t.Fatal(err) + } + c.act(t, kv, r) + run, _, _, _ := r.read(key) + if (run.Ended != nil) != c.ended { + t.Errorf("the run ended %v (%s), want %v", run.Ended != nil, run.Said, c.ended) + } + for prefix, want := range map[string]string{runBoundPrefix: c.bound, runUntakenPrefix: c.untaken} { + if held, _, _, _ := r.boundOf(prefix + key); held != want { + t.Errorf("%s holds the ask %q, want %q", prefix+key, held, want) + } + } + if c.journal != "" && !slices.ContainsFunc(*journal, func(l string) bool { return strings.Contains(l, c.journal) }) { + t.Errorf("the journal %v does not say %q", *journal, c.journal) + } + }) + } + // A-469y, A-470u: a step asked again that joins a running run not yet taken, beside a reader or in its pipeline's former + // reader's place, says its own wait held, as join does; the former reader's wait, and that of a reader leaving, ends at once + // superseded. + held := func(m string) said { return said{meshwants.EventWaitHeld, "", m} } + gone := func(m string) said { return said{meshwants.EventWaitEnded, "superseded", m} } + for _, c := range []struct { + name string + act func(t *testing.T, r *compositionRuns) + want []said + }{ + {"a rerun joins", func(t *testing.T, r *compositionRuns) { + again := reader("b") + again.Rerun = true + if _, err := r.rejoin(again, "main.3", "n1", "run-2"); err != nil { + t.Fatal(err) + } + }, []said{held("a"), held("b")}}, + {"a rerun takes its former place", func(t *testing.T, r *compositionRuns) { + again := reader("a2") + again.Pipeline, again.Module, again.Rerun = "main.3.a", "a2", true + if _, err := r.rejoin(again, "main.3", "n1", "run-2"); err != nil { + t.Fatal(err) + } + }, []said{held("a"), gone("a"), held("a2")}}, + {"a reader leaves", func(t *testing.T, r *compositionRuns) { + if _, err := r.join(reader("b"), "main.3", "n1", "run-2"); err != nil { + t.Fatal(err) + } + if _, err := r.leave("main.3.a", nil); err != nil { + t.Fatal(err) + } + }, []said{held("a"), held("b"), gone("a")}}, + } { + t.Run(c.name, func(t *testing.T) { + _, r, saids, _, _ := setup(t) + if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil { + t.Fatal(err) + } + c.act(t, r) + if !slices.Equal(*saids, c.want) { + t.Errorf("said %v, want %v", *saids, c.want) + } + }) + } + t.Run("taken up at the start", func(t *testing.T) { + _, r, _, _, _ := setup(t) + for _, c := range []struct{ id, machine string }{{"a", "n1"}, {"a", "n2"}, {"b", "n1"}} { + if _, err := r.join(reader(c.id), "main.3", c.machine, "run-"+c.machine); err != nil { + t.Fatal(err) + } + } + ended := now + _, _, _ = r.change(runKeyOf("main.3", "n2"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &ended, "passed"; return true }) + // A rerun's run carrying readers whose steps said their end (A-470l, A-470m): c passed and is not taken up, d is marked rerun. + _, _, _ = r.change(runKeyOf("main.3", "n1"), func(run *meshwants.CompositionRun) bool { + for _, id := range []string{"c", "d"} { + c := reader(id) + c.Ended, c.Rerun = &ended, id == "d" + run.Readers = append(run.Readers, c) + } + return true + }) + readers, err := r.readersOfOpen() + if err != nil || len(readers) != 3 || readers[0].ID != "a" || readers[1].ID != "b" || readers[2].ID != "d" { + t.Fatalf("the readers taken up: %+v %v", readers, err) + } + if keys, _ := r.keysOf("main.3", "a"); !slices.Equal(keys, []string{runKeyOf("main.3", "n1"), runKeyOf("main.3", "n2")}) { + t.Errorf("the runs of reader a: %v", keys) + } + // A-470q: a reader with no `ended` says its first end under its ask's id; d, marked rerun, says its end again under + // `.`. + ids := make(chan [2]string, 3) + steps := &compositionSteps{node: "n1", now: func() time.Time { return now }, logf: t.Logf, + resume: func(context.Context, string, meshwants.StepRun) stepOutcome { + return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed} + }, + say: func(_ string, body any, id string) error { + ids <- [2]string{body.(meshwants.StepEnded).ID, id} + return nil + }} + steps.takeUp(context.Background(), r) + got := map[string]string{} + for range 3 { + select { + case e := <-ids: + got[e[0]] = e[1] + case <-time.After(5 * time.Second): + t.Fatalf("the ends of the take-up: %v", got) + } + } + if want := map[string]string{"a": "a", "b": "b", "d": "d.run-n1"}; !maps.Equal(got, want) { + t.Errorf("the message ids of the take-up's ends: %v, want %v", got, want) + } + }) +} + +// missingRuns is runsMem whose listing misses one key, as a bucket's listing misses a key being written at that moment. +type missingRuns struct { + *runsMem + hide string +} + +func (k *missingRuns) keys() ([]string, error) { + all, err := k.runsMem.keys() + return slices.DeleteFunc(all, func(key string) bool { return key == k.hide }), err +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositions_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositions_tools.go new file mode 100644 index 00000000..df955f7d --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositions_tools.go @@ -0,0 +1,162 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// The composition runs' read and drop (design 47 G §728; A-469f, A-469u, A-469v): the seat mesh-wants' verbs `compositions` +// (class read) and `compositions-drop` (class act, §391's default callers), answered by the assigner's process, which keeps the +// runs and says the readers' ends; the module's tools `assigner_compositions` and `assigner_compositions_drop` call them. + +// The tools that call the two verbs. +const ( + compositionsTool = "assigner_compositions" + compositionsDropTool = "assigner_compositions_drop" +) + +// compositions answers each composition run the state keeps, under §719's field names plus its key, with the candidate and the +// machine named only those, and the state's fill against its cap. +func (r *compositionRuns) compositions(in meshwants.CompositionsInput) (meshwants.CompositionsAnswer, error) { + all, err := r.keys() + if err != nil { + return meshwants.CompositionsAnswer{}, fmt.Errorf("the state %s is not listed: %w", "composition-runs", err) + } + slices.Sort(all) + out := meshwants.CompositionsAnswer{Runs: []meshwants.CompositionRun{}} + for _, key := range all { + if !isRunKey(key) { + continue + } + run, _, ok, err := r.read(key) + if err != nil || !ok || (in.Candidate != "" && run.Candidate != in.Candidate) || (in.Machine != "" && run.Machine != in.Machine) { + continue + } + run.Key = key + out.Runs = append(out.Runs, run) + } + out.Fill, err = r.fill() + return out, err +} + +// drop drops one composition run by hand (§728): the act and why written in the hand-act log first (record), then a running run's +// ask dropped, each of its readers' waits stopped and, for each reader whose end is owed (no `ended`, or marked rerun), its end +// said could not run naming the drop, once (A-470r), and the key deleted with its bound keys. It answers the entry deleted and +// the readers it ended. +func (c *compositionSteps) drop(r *compositionRuns, record func(meshhandactsclient.Request) error, caller string, in meshwants.CompositionsDropInput) (meshwants.CompositionsDropAnswer, error) { + why := strings.TrimSpace(in.Why) + if in.Candidate == "" || in.Machine == "" || why == "" { + return meshwants.CompositionsDropAnswer{}, errors.New("name the candidate, the machine and why the composition run is dropped") + } + key := runKeyOf(in.Candidate, in.Machine) + run, _, ok, err := r.read(key) + if err != nil { + return meshwants.CompositionsDropAnswer{}, err + } + if !ok { + return meshwants.CompositionsDropAnswer{}, fmt.Errorf("no composition run of %s on %s is kept", in.Candidate, in.Machine) + } + if err := record(meshhandactsclient.Request{What: meshwants.VerbCompositionsDrop + " " + key, Why: why + " (" + caller + ")", Cause: "compositions-drop"}); err != nil { + return meshwants.CompositionsDropAnswer{}, fmt.Errorf("the hand act was not recorded, so nothing was dropped: %w", err) + } + said := fmt.Sprintf("%s: the composition run %s was dropped by hand: %s", in.Machine, key, why) + answer := meshwants.CompositionsDropAnswer{Run: run} + answer.Run.Key = key + if run.Ended == nil { + if r.drop != nil { + if err := r.drop(run.Ask, said); err != nil { + r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err) + } + } + ids := make([]string, 0, len(run.Readers)) + for _, reader := range run.Readers { + ids = append(ids, reader.ID) + } + c.leave(ids) // their waits say no end of their own + // The run ends without moving its readers: the drop says each owed end itself, once (A-470r). + if _, err := r.endRunMoving(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil, r.untaken(key, run.Ask), false); err != nil { + return answer, err + } + shared := []meshwants.Shared{{Machine: run.Machine, Run: key, Ask: run.Ask}} + for _, reader := range run.Readers { + if reader.Ended != nil && !reader.Rerun { + continue // it ended passed already and keeps its verdict (A-470r) + } + again := "" // a reader whose step said its end already says it again under the dropped run's ask (A-470q) + if reader.Ended != nil { + again = run.Ask + } + c.sayEnd(reader.ID, stepRunOf(reader), nil, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: said, Shared: shared}, again) + answer.Ended = append(answer.Ended, reader) + } + } + // The dropped run's key goes while it holds that run: a run started on it since stands. + err = r.deleteRun(key, func(c meshwants.CompositionRun) bool { return c.Ask == run.Ask }) + r.heard() + return answer, err +} + +// compositionVerbs are the process's handlers of the seat mesh-wants' verbs compositions and compositions-drop. +func compositionVerbs(r *compositionRuns, c *compositionSteps, record func(meshhandactsclient.Request) error) map[string]process.SeatHandler { + return map[string]process.SeatHandler{ + meshwants.VerbCompositions: func(_ string, raw json.RawMessage) (any, error) { + var in meshwants.CompositionsInput + if len(raw) > 0 { + if err := json.Unmarshal(raw, &in); err != nil { + return nil, fmt.Errorf("the arguments are not JSON: %w", err) + } + } + return r.compositions(in) + }, + meshwants.VerbCompositionsDrop: func(caller string, raw json.RawMessage) (any, error) { + var in meshwants.CompositionsDropInput + if err := json.Unmarshal(raw, &in); err != nil { + return nil, fmt.Errorf("the arguments are not JSON: %w", err) + } + return c.drop(r, record, caller, in) + }, + } +} + +// compositionsTools are the module's tools of §728's names, each calling its verb (A-469f). +func compositionsTools(ask func(string, any) (json.RawMessage, error)) []stdio.Tool { + call := func(verb string, args map[string]any, names ...string) (any, error) { + in := map[string]any{} + for _, n := range names { + if v := arg(args, n); v != "" { + in[n] = v + } + } + raw, err := ask("seat:"+meshwants.Seat+"."+verb, in) + if err != nil { + return nil, err + } + var out any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("%s answered what is not JSON: %w", verb, err) + } + return out, nil + } + return []stdio.Tool{ + {Name: compositionsTool, Description: "The composition runs the assigner keeps, one per candidate and machine (design 47 G §719, §728): each with " + + "its program, input digest, ask, readers, outcome and verdict, and the state's fill against its cap; with candidate or machine, only those. Read-only.", + Input: schema(map[string]string{"candidate": "the candidate's key: ., or run-", "machine": "a machine"}, nil), + Run: func(args map[string]any) (any, error) { + return call(meshwants.VerbCompositions, args, "candidate", "machine") + }}, + {Name: compositionsDropTool, Description: "Drop one composition run by hand (design 47 G §728): written in the hand-act log first, then its ask " + + "is dropped, each reader ended could not run naming the drop, and the key deleted.", + Input: schema(map[string]string{"candidate": "the candidate's key", "machine": "the machine", "why": "why the run is dropped"}, []string{"candidate", "machine", "why"}), + Run: func(args map[string]any) (any, error) { + return call(meshwants.VerbCompositionsDrop, args, "candidate", "machine", "why") + }}, + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositions_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositions_tools_test.go new file mode 100644 index 00000000..b1688be1 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositions_tools_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" +) + +// Design 47 G §728, A-469f, A-469u, A-469v: compositions answers each run with its key and the fill, only those of a candidate or +// machine when named; compositions-drop writes the hand act first (nothing is dropped when it is not recorded), drops a running +// run's ask, ends each reader could not run naming the drop and deletes the key with its bound keys; the tools call the verbs. +func TestTheCompositionRunsAreReadAndDroppedByTheirVerbs(t *testing.T) { + now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC) + kv := newRunsMem() + r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf) + var drops []string + r.drop = func(ask, why string) error { drops = append(drops, ask); return nil } + ends := newSaidEvents() + var endIDs []string // the message id of each end said + c := &compositionSteps{node: "n1", now: func() time.Time { return now }, logf: t.Logf, say: func(event string, body any, id string) error { + endIDs = append(endIDs, id) + return ends.say(event, body, id) + }} + moves := 0 + move := func(meshwants.CompositionRun) { moves++ } + r.moved.Store(&move) + for _, m := range []string{"n1", "n2"} { + if _, err := r.join(readerOf("ask-a", meshwants.StepRun{Pipeline: "main.3.a", Step: meshwants.StepRunStepComposition, Module: "a"}), "main.3", m, "run-"+m); err != nil { + t.Fatal(err) + } + } + // A-470r: n2's run carries a reader that ended passed (p) and one marked rerun (q); only q's end is owed, said once. + _, _, _ = r.change(runKeyOf("main.3", "n2"), func(run *meshwants.CompositionRun) bool { + for _, id := range []string{"ask-p", "ask-q"} { + c := readerOf(id, meshwants.StepRun{Pipeline: "main.3." + id, Step: meshwants.StepRunStepComposition, Module: id}) + c.Ended, c.Rerun = &now, id == "ask-q" + run.Readers = append(run.Readers, c) + } + return true + }) + var recorded []meshhandactsclient.Request + record := func(q meshhandactsclient.Request) error { recorded = append(recorded, q); return nil } + verbs := compositionVerbs(r, c, record) + out, err := verbs[meshwants.VerbCompositions]("op", json.RawMessage(`{"machine":"n2"}`)) + read, _ := out.(meshwants.CompositionsAnswer) + if err != nil || len(read.Runs) != 1 || read.Runs[0].Key != runKeyOf("main.3", "n2") || read.Runs[0].Ask != "run-n2" || len(read.Runs[0].Readers) != 3 || + read.Fill.Store != "composition-runs" || read.Fill.Cap != 16<<20 || read.Fill.Bytes == 0 { + t.Fatalf("compositions = %+v %v", out, err) + } + refuse := compositionVerbs(r, c, func(meshhandactsclient.Request) error { return errors.New("no holder") }) + if _, err := refuse[meshwants.VerbCompositionsDrop]("op", json.RawMessage(`{"candidate":"main.3","machine":"n2","why":"stuck"}`)); err == nil || + !strings.Contains(err.Error(), "nothing was dropped") || len(drops) != 0 { + t.Fatalf("a drop not recorded: %v, drops %v", err, drops) + } + out, err = verbs[meshwants.VerbCompositionsDrop]("op@n1", json.RawMessage(`{"candidate":"main.3","machine":"n2","why":"stuck"}`)) + dropped, _ := out.(meshwants.CompositionsDropAnswer) + if err != nil || dropped.Run.Key != runKeyOf("main.3", "n2") || len(dropped.Ended) != 2 || dropped.Ended[0].ID != "ask-a" || dropped.Ended[1].ID != "ask-q" || + moves != 0 || len(endIDs) != 2 || endIDs[0] != "ask-a" || endIDs[1] != "ask-q.run-n2" || + len(recorded) != 1 || recorded[0].Why != "stuck (op@n1)" || len(drops) != 1 || drops[0] != "run-n2" { + t.Fatalf("compositions-drop = %+v %v, recorded %+v, drops %v, moves %d, ends %v", out, err, recorded, drops, moves, endIDs) + } + end, _ := ends.said[meshwants.EventStepEnded+".n1"].(meshwants.StepEnded) + if end.ID != "ask-q" || end.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(end.Said, "dropped by hand: stuck") || + len(end.Shared) != 1 || end.Shared[0].Ask != "run-n2" { + t.Errorf("the reader's end %+v", end) + } + if keys, _ := kv.keys(); len(keys) != 2 || keys[0] != runKeyOf("main.3", "n1") { + t.Errorf("keys after the drop %v, want n1's run and its untaken key", keys) + } + // The tools call the verbs with what they were given. + var asked []string + tools := compositionsTools(func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + asked = append(asked, key+" "+string(raw)) + return json.RawMessage(`{"runs":[],"fill":{"store":"composition-runs","bytes":0,"cap":1}}`), nil + }) + for _, tool := range tools { + if _, err := tool.Run(map[string]any{"candidate": "main.3", "machine": "n1", "why": "x"}); err != nil { + t.Fatal(err) + } + } + if len(tools) != 2 || tools[0].Name != compositionsTool || tools[1].Name != compositionsDropTool || + asked[0] != `seat:mesh-wants.compositions {"candidate":"main.3","machine":"n1"}` || asked[1] != `seat:mesh-wants.compositions-drop {"candidate":"main.3","machine":"n1","why":"x"}` { + t.Errorf("the tools asked %v", asked) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go index 48cedf7e..3f91b28d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go @@ -23,6 +23,8 @@ type stepOutcome struct { Outcome string Said string Warnings []string + // Shared names per machine composed the composition run the step read and its ask (design 47 G §723, §729). + Shared []meshwants.Shared } // compositionSteps takes the composition steps and says their ends. @@ -30,14 +32,21 @@ type compositionSteps struct { node string // say says one event of the seat mesh-wants (wantsPublisher). say publisher - // run runs one step and answers what it came to. - run func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome - now func() time.Time - logf func(string, ...any) + // run runs one step and answers what it came to; resume judges a step taken up at the start from its runs (takeUp). + run func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome + resume func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome + // ended writes, once the step asked under id said its end, that time on its reader in each run naming it (A-470m); nil + // writes none. + ended func(id string, run meshwants.StepRun, at time.Time) + now func() time.Time + logf func(string, ...any) // asker is the name the step's progress names as its asker, read when it is said (stepAskerOf). asker func() string // ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own. ticks func(d time.Duration) (<-chan time.Time, func()) + mu sync.Mutex + // stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave). + stops map[string]context.CancelFunc } // stepAskerOf is the asker a composition step's progress names: the one caller the seat mesh-wants' definition lists for its @@ -69,10 +78,27 @@ func stepAskerOf(definition func() (seatdef.Definition, bool)) func() string { // progressEvery is how often a running composition step says its progress (design 47 G §496, A-262). const progressEvery = time.Minute -// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context. +// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context, and the +// composition runs it waits on (§723, §729). type stepPhase struct { - mu sync.Mutex - phase string + mu sync.Mutex + phase string + shared []meshwants.Shared +} + +// setShared names the runs the step ctx runs waits on, when it runs one. +func setShared(ctx context.Context, shared []meshwants.Shared) { + if p, ok := ctx.Value(phaseKey{}).(*stepPhase); ok { + p.mu.Lock() + p.shared = shared + p.mu.Unlock() + } +} + +func (p *stepPhase) waitsOn() []meshwants.Shared { + p.mu.Lock() + defer p.mu.Unlock() + return p.shared } type phaseKey struct{} @@ -100,23 +126,97 @@ func (c *compositionSteps) take(ctx context.Context, id string, data []byte) boo c.logf("a step ask that is no composition step with an id is left: %v", err) return false } - go c.runOne(ctx, id, run) + go c.say1(ctx, id, run, c.run, "") return true } -// runOne runs one step and says its end. -func (c *compositionSteps) runOne(ctx context.Context, id string, run meshwants.StepRun) { +// say1 runs one step with how and says its end, naming in `shared` the runs it read (§723); again is, for an end said again, the +// ask of the run whose end makes it say again (sayEnd). A step whose wait was ended by its pipeline's supersede or cancel (leave) +// says nothing: the runner ended it already. +func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome, again string) { started := c.now().UTC() + ctx, stopWait := context.WithCancel(ctx) + c.mu.Lock() + if c.stops == nil { + c.stops = map[string]context.CancelFunc{} + } + if c.stops[id] != nil { // the step asked under id is running already: it says its own end + c.mu.Unlock() + stopWait() + return + } + c.stops[id] = stopWait + c.mu.Unlock() + defer func() { + c.mu.Lock() + delete(c.stops, id) + c.mu.Unlock() + stopWait() + }() phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch} stop := c.sayProgress(id, phase) - o := c.run(context.WithValue(ctx, phaseKey{}, phase), id, run) + o := how(context.WithValue(ctx, phaseKey{}, phase), id, run) stop() + if ctx.Err() != nil { + c.logf("the composition step %s of %s says no end: its wait ended (%v)", id, run.Pipeline, context.Cause(ctx)) + return + } + c.sayEnd(id, run, &started, o, again) +} + +// sayEnd says the end of the step asked under id as the seat's step-ended., its id the ask's; once said, its time is +// written as its reader's `ended` (A-470m). Its first end's message id is the ask's id; an end said again is published under +// `.`, again the ask of the run whose end makes it say again, so the bus's deduplication never drops it (A-470q). +func (c *compositionSteps) sayEnd(id string, run meshwants.StepRun, started *time.Time, o stepOutcome, again string) { ended := c.now().UTC() + if started == nil { + started = &ended + } end := meshwants.StepEnded{V: 1, ID: id, Pipeline: run.Pipeline, Step: meshwants.StepEndedStepComposition, Module: run.Module, Commit: run.Commit, Fingerprint: run.Fingerprint, Node: c.node, Outcome: o.Outcome, Said: o.Said, Warnings: o.Warnings, - Rerun: run.Rerun, By: run.By, Started: &started, Ended: &ended} - if err := c.say(meshwants.EventStepEnded+"."+c.node, end, id); err != nil { + Rerun: run.Rerun, By: run.By, Started: started, Ended: &ended, Shared: o.Shared} + msgID := id + if again != "" { + msgID = id + "." + again + } + if err := c.say(meshwants.EventStepEnded+"."+c.node, end, msgID); err != nil { c.logf("saying the end of the composition step %s: %v", id, err) + return + } + if c.ended != nil { + c.ended(id, run, ended) + } +} + +// move says the end again of each reader a rerun carried into an ended run, marked rerun (§727, A-470d): its machines judged +// from the candidate's keys naming it, the run's new ask in `shared`. A reader running here says its own end. +func (c *compositionSteps) move(ctx context.Context) func(run meshwants.CompositionRun) { + return func(run meshwants.CompositionRun) { + for _, reader := range run.Readers { + if !reader.Rerun || c.running(reader.ID) { + continue + } + c.logf("the composition step %s of %s moves with the composition run of %s (ask %s)", reader.ID, reader.Pipeline, run.Machine, run.Ask) + go c.say1(ctx, reader.ID, stepRunOf(reader), c.resume, run.Ask) // said again under the moved run's ask (A-470q) + } + } +} + +// running is whether the step asked under id runs here. +func (c *compositionSteps) running(id string) bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.stops[id] != nil +} + +// leave ends the waits of the steps asked under ids (§723: a superseded or cancelled step leaves its runs). +func (c *compositionSteps) leave(ids []string) { + c.mu.Lock() + defer c.mu.Unlock() + for _, id := range ids { + if stop := c.stops[id]; stop != nil { + stop() + } } } @@ -139,7 +239,7 @@ func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func() return case <-tick: at := c.now().UTC() - p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at} + p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at, Shared: phase.waitsOn()} if err := c.say(meshwants.EventStepProgress+"."+c.node, p, id+".progress."+at.Format(time.RFC3339Nano)); err != nil { c.logf("saying the progress of the composition step %s: %v", id, err) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go index 140f2382..230d16f7 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go @@ -168,7 +168,8 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T) t.Fatal("the composition step said no end") } if e.ID != "step-ask-1" || e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose") || - strings.Contains(e.Said, "n1") || len(e.Warnings) != 1 || !strings.Contains(e.Warnings[0], "n3 is away") { + strings.Contains(e.Said, "n1") || len(e.Warnings) != 2 || !strings.Contains(e.Warnings[0], "n3 is away") || + e.Warnings[1] != "n2: the composer names no culprits" { t.Fatalf("the step ended %+v", e) } if programs["n1"] != engineCommit || programs["n2"] != engineCommit { @@ -180,9 +181,24 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T) t.Error("the machine away was asked for its input") } } - keys, _ := b.buckets[buckets.Inputs()].Keys(b.ctx) - if len(keys) != 0 { - t.Errorf("the inputs stay kept past their step-ended: %v", keys) + // The run's end comes before the input's delete (A-470o, A-470p): the inputs go after the step's end, heard by their watch. + inputs := b.buckets[buckets.Inputs()] + w, err := inputs.WatchAll(b.ctx) + if err != nil { + t.Fatal(err) + } + defer func() { _ = w.Stop() }() + deadline := time.After(10 * time.Second) + for { + keys, _ := inputs.Keys(b.ctx) + if len(keys) == 0 { + break + } + select { + case <-w.Updates(): + case <-deadline: + t.Fatalf("the inputs stay kept past their step-ended: %v", keys) + } } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go index 7467700d..07e25cb8 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "git.novox.be/novox/mesh-sdk/go/seatdef" + "slices" + "strings" "sync" "testing" "time" @@ -40,7 +42,8 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) { c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf, run: func(_ context.Context, id string, run meshwants.StepRun) stepOutcome { asked = run - return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"}} + return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"}, + Shared: []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}}} }} other, _ := json.Marshal(map[string]any{"pipeline": "main.4.m", "step": "build", "module": "m"}) if c.take(context.Background(), "ask-0", other) { @@ -67,7 +70,8 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) { if asked.Module != "m" || end.ID != "ask-1" || end.Pipeline != "main.4.m" || end.Step != meshwants.StepEndedStepComposition || end.Module != "m" || end.Commit != "c0ffee" || end.Fingerprint != "fp1" || end.Node != "n1" || end.Outcome != meshwants.StepEndedOutcomeFailed || end.Said != "a and b own /etc/x" || len(end.Warnings) != 1 || - !end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil { + !end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil || + len(end.Shared) != 1 || end.Shared[0] != (meshwants.Shared{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}) { // §723, §729 t.Fatalf("step-ended %+v", end) } } @@ -105,6 +109,7 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) { }, run: func(ctx context.Context, _ string, _ meshwants.StepRun) stepOutcome { setPhase(ctx, meshwants.StepProgressEventPhaseRun) + setShared(ctx, []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}}) <-release return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed} }} @@ -122,7 +127,8 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) { said.mu.Lock() p, ok := said.said[progress].(meshwants.StepProgressEvent) said.mu.Unlock() - if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) { + if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) || + len(p.Shared) != 1 || p.Shared[0].Ask != "run-1" { // the runs the step waits on (§723) t.Fatalf("step-progress %+v", p) } close(release) @@ -135,3 +141,70 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) { t.Fatal("the ticker is not stopped when the step has ended") } } + +// Design 47 G §723, A-469n, A-469z: a superseded or cancelled pipeline's composition step leaves the runs it reads and says no end; +// a run another pipeline's step reads goes on for it; a run left with no reader has its ask dropped and ends could not run, "no +// reader is left", so a later step starts a new run. +func TestASupersededStepLeavesItsRuns(t *testing.T) { + now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC) + r := newCompositionRuns(newRunsMem(), defaultRunSettings, func() time.Time { return now }, t.Logf) + var drops []string + r.drop = func(ask, why string) error { drops = append(drops, ask+": "+why); return nil } + step := func(pipeline string) meshwants.StepRun { + return meshwants.StepRun{Pipeline: pipeline, Step: meshwants.StepRunStepComposition, Module: pipeline, Branch: "main", Revision: 3} + } + said := newSaidEvents() + c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return now }, logf: t.Logf, + run: func(ctx context.Context, id string, s meshwants.StepRun) stepOutcome { + j, err := r.join(readerOf(id, s), "main.3", "n1", "run-"+id) + if err != nil { + t.Error(err) + } + r.await(ctx, []string{j.key}) + return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed} + }} + body := func(s meshwants.StepRun) []byte { raw, _ := json.Marshal(s); return raw } + for _, p := range []string{"main.3.a", "main.3.b"} { + if !c.take(context.Background(), "ask-"+p, body(step(p))) { + t.Fatal("not taken") + } + } + key := runKeyOf("main.3", "n1") + for { + wake := r.wake() + if run, _, _, _ := r.read(key); len(run.Readers) == 2 { + break + } + <-wake + } + ids, err := r.leave("main.3.a", c.leave) + run, _, _, _ := r.read(key) + if err != nil || !slices.Equal(ids, []string{"ask-main.3.a"}) || run.Ended != nil || len(run.Readers) != 1 || len(drops) != 0 { + t.Fatalf("after a's supersede: %v %v, run %+v, drops %v: want the run going on for b", ids, err, run, drops) + } + _, err = r.leave("main.3.b", c.leave) + run, _, _, _ = r.read(key) + if err != nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || !strings.Contains(run.Said, "no reader is left") || len(drops) != 1 || + !strings.HasPrefix(drops[0], run.Ask+": no reader is left") { + t.Fatalf("after b's supersede: %v, run %+v, drops %v", err, run, drops) + } + deadline := time.After(5 * time.Second) + for { + c.mu.Lock() + n := len(c.stops) + c.mu.Unlock() + if n == 0 { + break + } + select { + case <-deadline: + t.Fatal("the steps that left still wait") + case <-time.After(time.Millisecond): + } + } + said.mu.Lock() + defer said.mu.Unlock() + if _, ok := said.said[meshwants.EventStepEnded+".n1"]; ok { + t.Errorf("a step that left says its end: %v", said.said) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs.go index b32b5bd0..4cb138c9 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs.go @@ -25,14 +25,17 @@ import ( "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/stateobjects" + + "mesh-delivery-assigner/internal/buckets" ) // A step's input by its digest (design 47 G §354, §712, A-237, A-243): the assigner keeps the input of each composition step -// request it asked a build seat to run in its state `inputs`, one key `