diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go index 2e763747..52c52263 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go @@ -10,7 +10,6 @@ import ( "errors" "fmt" "slices" - "sync" "time" nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" @@ -18,8 +17,8 @@ import ( meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" ) -// The inputs of a composition step (design 47 G §354, A-243, A-252): one request per machine, each carrying that machine's input, -// asked of the machine's node-engine by the seat's verb compose-candidate. +// The inputs of a composition run (design 47 G §354, §718, A-243, A-252): one request per candidate and machine, carrying that +// machine's input, asked of the machine's node-engine by the seat's verb compose-candidate when the run starts. // composeCandidateKey is the seat verb's call key, asked of one node. const composeCandidateKey = "seat:" + ne.Seat + "." + ne.VerbComposeCandidate @@ -80,37 +79,14 @@ type compositionInputs struct { bound time.Duration } -// gather asks each machine of the step's module and branch for its input with the step's candidate key, and answers every machine in -// name order: one away (away) is not asked; a node that is silent, unready or answers no input has Why set; no machine at all is an -// error. The run keeps each request built from an input under its run ask (compositionRun.machine). -func (c *compositionInputs) gather(ctx context.Context, run meshwants.StepRun, away func(node string) bool) ([]nodeInput, error) { - key, err := candidateKeyOf(run) - if err != nil { - return nil, err - } - nodes := c.machines(run.Module, run.Branch) - if len(nodes) == 0 { - return nil, fmt.Errorf("no node places %s on the branch %q, so the composition step has no machine to compose", run.Module, run.Branch) - } +// of asks the machine node for its input with the candidate's key (§718): asked only by the composition step that starts the +// machine's composition run, once per candidate and machine. A node that is silent, unready or answers no input has Why set. +func (c *compositionInputs) of(ctx context.Context, node, candidate string) nodeInput { bound := c.bound if bound <= 0 { bound = nec.AnswerBound } - out := make([]nodeInput, len(nodes)) - var wg sync.WaitGroup - for i, node := range nodes { - if away != nil && away(node) { - out[i] = nodeInput{Node: node, Away: true} - continue - } - wg.Add(1) - go func() { - defer wg.Done() - out[i] = c.one(ctx, bound, node, key) - }() - } - wg.Wait() - return out, nil + return c.one(ctx, bound, node, candidate) } // one asks one node for its input. diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go index 78fe48e1..d403de52 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go @@ -68,7 +68,7 @@ func TestTheMachinesOfAStep(t *testing.T) { // §354, A-243, A-252, §64: each machine is asked compose-candidate with the step's key and input true, its numbers as written; a // silent or unready node says why, and a machine away is not asked. func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) { - // gather asks the machines side by side, so the record of what each was asked is guarded. + // the machines are asked side by side, so the record of what each was asked is guarded. var mu sync.Mutex asked := map[string]string{} call := func(key string, body any) (json.RawMessage, error) { @@ -86,7 +86,7 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) { return json.RawMessage(`{"candidate":"run-r1","node":"` + node + `","verdict":"composed","input":{"n":12345678901234567890,"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil } c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14", "novox", "shanks"} }, callFile: call, bound: time.Second} - got, err := c.gather(context.Background(), meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" }) + got, err := gatherAll(c, meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" }) if err != nil || len(got) != 4 { t.Fatalf("%+v %v", got, err) } @@ -105,7 +105,7 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) { t.Errorf("the machine away is asked or not said away: %+v", got[3]) } none := &compositionInputs{machines: func(string, string) []string { return nil }, callFile: call} - if _, err := none.gather(context.Background(), meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil { + if _, err := gatherAll(none, meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil { t.Error("a step with no machine is gathered") } } @@ -124,7 +124,7 @@ func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) { return json.RawMessage(answer + `,"file":"` + file + `"}`), nil } c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14"} }, callFile: call, bound: time.Second} - got, err := c.gather(context.Background(), meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(string) bool { return false }) + got, err := gatherAll(c, meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(string) bool { return false }) if err != nil || len(got) != 2 { t.Fatalf("%+v %v", got, err) } @@ -135,3 +135,28 @@ func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) { t.Errorf("g14: %+v", got[1]) } } + +// gatherAll asks each machine of a step for its input as each run's start does (compositionInputs.of), side by side; a machine away +// is not asked; no machine is an error. +func gatherAll(c *compositionInputs, run meshwants.StepRun, away func(string) bool) ([]nodeInput, error) { + key, err := candidateKeyOf(run) + if err != nil { + return nil, err + } + nodes := c.machines(run.Module, run.Branch) + if len(nodes) == 0 { + return nil, errors.New("no machine") + } + out := make([]nodeInput, len(nodes)) + var wg sync.WaitGroup + for i, node := range nodes { + if away != nil && away(node) { + out[i] = nodeInput{Node: node, Away: true} + continue + } + wg.Add(1) + go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key) }() + } + wg.Wait() + return out, nil +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go index b14d3ce7..76394244 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go @@ -21,12 +21,14 @@ import ( "git.novox.be/novox/mesh-sdk/go/keys" ) -// The run of a composition step (design 47 G §354, §63, §64; A-226, A-238, A-240, A-243, A-252, A-254, A-261): one -// composer.Request per machine, depth compose, the one machine and one case `change`, its manifests, verdicts and seats from the -// step's catalog revision read through mesh-catalog's `revision`; one run ask per machine of the seat node-build-go, naming no -// head, its program the node-engine build's catalog entry with the composer's package; the run's end heard on the build seat's -// step-ended of that ask, its answer (a composer.Verdict) read inline from `output`. A machine the watchman said away -// (machine-away) is not composed and passes with a warning (§64). +// The run of a composition step (design 47 G §354, §63, §64, §718; A-226, A-238, A-240, A-252, A-254, A-261): the step reads one +// composition run per machine of its candidate (compositionruns.go), and the step that starts a machine's run composes it: one +// compose-candidate ask of the node, one composer.Request, depth compose, the one machine and one case `change`, its manifests, +// verdicts and seats from the step's catalog revision read through mesh-catalog's `revision`, kept once in `inputs`; one run ask +// of the seat node-build-go, naming no head, its program the node-engine build's catalog entry with the composer's package, +// chosen once and recorded in the run. The run's end is heard on the build seat's step-ended of that ask, its answer (a +// composer.Verdict) read from `output`, and written in the run's key, whose change every reading step hears. A machine the +// watchman said away (machine-away) is not composed and passes with a warning (§64). const ( // runAsker is the assigner's name as an asker of a build seat, its seat's own name: the last token of the subject its run @@ -41,42 +43,6 @@ const ( seatDefinitionsKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbSeatDefinitions ) -// runEnds hands each build seat's step-ended of an ask to the composition run that waits for it (A-240), by the ask's id. -type runEnds struct { - mu sync.Mutex - waits map[string]chan buildstep.StepEnded -} - -// expect is the end of the ask id, once heard; called before the ask is put, so no end is missed. -func (r *runEnds) expect(id string) <-chan buildstep.StepEnded { - r.mu.Lock() - defer r.mu.Unlock() - if r.waits == nil { - r.waits = map[string]chan buildstep.StepEnded{} - } - ch := make(chan buildstep.StepEnded, 1) - r.waits[id] = ch - return ch -} - -// forget stops waiting for the ask id. -func (r *runEnds) forget(id string) { - r.mu.Lock() - delete(r.waits, id) - r.mu.Unlock() -} - -// heard hands an end to the run that waits for its ask; one nobody waits for is another asker's, left. -func (r *runEnds) heard(e buildstep.StepEnded) { - r.mu.Lock() - ch := r.waits[e.Ask] - delete(r.waits, e.Ask) - r.mu.Unlock() - if ch != nil { - ch <- e - } -} - // awayNodes are the machines the watchman said away (machine-away of the seat mesh-watchdogs, §389) and not back since. type awayNodes struct { mu sync.Mutex @@ -121,8 +87,7 @@ type compositionRun struct { running func(node, module string) (ne.Running, bool) away *awayNodes // put puts a run ask on the build seat's check queue under the assigner's name. - put func(ctx context.Context, a buildstep.StepAsk) error - ends *runEnds + put func(ctx context.Context, a buildstep.StepAsk) error // reads shares the catalog reads among steps (compositionreads.go); sleep waits between two tries of a read (nil: a timer). reads *compositionReads sleep func(ctx context.Context, d time.Duration) error @@ -139,41 +104,49 @@ type machineEnd struct { judged bool } -// run runs one composition step and answers what it came to: failed when a machine does not compose, could not run when a machine -// could not be composed or was not judged, else passed with a warning per machine away. -func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRun) stepOutcome { - half, err := r.revision(ctx, run) - if err != nil { - return couldNotRun("the step's catalog revision is not read: " + err.Error()) - } - seats, err := r.seats(ctx, half.Revision.Definitions) +// run runs one composition step and answers what it came to: per machine not away it joins or starts the candidate's +// composition run (§718), waits for each run's end in its key, and judges each machine from it; failed when a machine does not +// compose, could not run when a machine could not be composed or was not judged, else passed with a warning per machine away. +func (r *compositionRun) run(ctx context.Context, id string, step meshwants.StepRun) stepOutcome { + candidate, err := candidateKeyOf(step) if err != nil { return couldNotRun(err.Error()) } - manifests, verdicts, err := caseOf(half) - if err != nil { - return couldNotRun(err.Error()) + nodes := r.inputs.machines(step.Module, step.Branch) + if len(nodes) == 0 { + return couldNotRun(fmt.Sprintf("no node places %s on the branch %q, so the composition step has no machine to compose", step.Module, step.Branch)) } - programOf, err := r.programs(ctx, half) - if err != nil { - return couldNotRun(err.Error()) - } - inputs, err := r.inputs.gather(ctx, run, r.away.is) - if err != nil { - return couldNotRun(err.Error()) + reader := readerOf(id, step) + prep := &runPrep{r: r, step: step} + var ends []machineEnd + var keys []string + for _, node := range nodes { + if r.away.is(node) { + ends = append(ends, machineEnd{warning: node + " is away (the watchman's machine-away), so it is not composed"}) + continue + } + j, err := r.runs.join(reader, candidate, node, buildstep.NewID()) + if err != nil { + ends = append(ends, machineEnd{couldNot: node + ": " + err.Error()}) + continue + } + if j.started { + go r.start(ctx, j.key, j.run, prep) + } + keys = append(keys, j.key) } setPhase(ctx, meshwants.StepProgressEventPhaseRun) - ends := make([]machineEnd, len(inputs)) - var wg sync.WaitGroup - for i, in := range inputs { - wg.Add(1) - go func() { - defer wg.Done() - ends[i] = r.machine(ctx, in, buildstep.NewID(), programOf, manifests, verdicts, seats) - }() - } - wg.Wait() + runs := r.runs.await(ctx, keys) setPhase(ctx, meshwants.StepProgressEventPhaseReport) + for _, key := range keys { + ends = append(ends, judgeRun(runs[key])) + } + return outcomeOf(ends) +} + +// outcomeOf is a step's outcome from its machines: failed before could not run before passed (§354); a step that judged no +// machine never passes (A-467a), each machine not judged named with why. +func outcomeOf(ends []machineEnd) stepOutcome { var out stepOutcome var failed, couldNot []string judged := 0 @@ -190,7 +163,7 @@ func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRu out.Outcome, out.Said = meshwants.StepEndedOutcomeFailed, strings.Join(failed, "; ") case len(couldNot) > 0: out.Outcome, out.Said = meshwants.StepEndedOutcomeCouldNotRun, strings.Join(couldNot, "; ") - case judged == 0: // a step that judged no machine never passes (A-467a): each machine not judged is named with why + case judged == 0: said := "no machine was judged" if len(out.Warnings) == 0 { said += ": the step has no machine" @@ -215,82 +188,158 @@ func couldNotRun(said string) stepOutcome { return stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: said} } -// machine composes one machine: its request kept under its ask, the ask put, its end heard and its verdict read. -func (r *compositionRun) machine(ctx context.Context, in nodeInput, ask string, programOf func(node string) (buildstep.RunProgram, error), - manifests map[string]composer.Manifest, verdicts map[string]json.RawMessage, seats []json.RawMessage) machineEnd { - node := in.Node - switch { - case in.Away: - return machineEnd{warning: node + " is away (the watchman's machine-away), so it is not composed"} - case in.Why != "": - return machineEnd{couldNot: node + " gave no composition input: " + in.Why} +// runPrep is what the step that starts a run reads of its catalog revision, once for every machine it starts: the seats, the +// case's manifests and verdicts, and the composer program per machine. +type runPrep struct { + r *compositionRun + step meshwants.StepRun + once sync.Once + // read once + seats []json.RawMessage + manifests map[string]composer.Manifest + verdicts map[string]json.RawMessage + programOf func(node string) (buildstep.RunProgram, string, error) + err error +} + +func (p *runPrep) get(ctx context.Context) error { + p.once.Do(func() { + half, err := p.r.revision(ctx, p.step) + if err != nil { + p.err = errors.New("the step's catalog revision is not read: " + err.Error()) + return + } + if p.seats, p.err = p.r.seats(ctx, half.Revision.Definitions); p.err != nil { + return + } + if p.manifests, p.verdicts, p.err = caseOf(half); p.err != nil { + return + } + p.programOf, p.err = p.r.programs(ctx, half) + }) + return p.err +} + +// start composes the machine of the run it started (§718): the node's input of this moment, its request kept once in `inputs`, +// the composer program chosen and recorded with the request's digest, and one run ask put. A fault ends the run could not run. +func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep) { + node, ask := run.Machine, run.Ask + end := func(said string) { + if _, err := r.runs.end(key, ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil); err != nil { + r.logf("the composition run %s is not ended: %v", key, err) + } } - program, err := programOf(node) + if err := prep.get(ctx); err != nil { + end(node + ": " + err.Error()) + return + } + in := r.inputs.of(ctx, node, run.Candidate) + if in.Why != "" { + end(node + " gave no composition input: " + in.Why) + return + } + program, engineFp, err := prep.programOf(node) if err != nil { - return machineEnd{couldNot: node + ": " + err.Error()} + end(node + ": " + err.Error()) + return } req := composer.Request{V: composer.Version, Input: in.Body, Depth: composer.DepthCompose, Machines: []string{node}, - Cases: []composer.Case{{Name: changeCase, Manifests: manifests, Verdicts: verdicts}}, Seats: seats} + Cases: []composer.Case{{Name: changeCase, Manifests: prep.manifests, Verdicts: prep.verdicts}}, Seats: prep.seats} raw, err := json.Marshal(req) if err != nil { - return machineEnd{couldNot: node + ": its request cannot be encoded: " + err.Error()} + end(node + ": its request cannot be encoded: " + err.Error()) + return } - if _, err := r.keeper.keep(ask, raw); err != nil { + digest, err := r.keeper.keep(ask, raw) + if err != nil { r.logf("the composition request of %s is refused: %v", node, err) - return machineEnd{couldNot: node + ": its request is not kept: " + err.Error()} + end(node + ": its request is not kept: " + err.Error()) + return } ref, err := buildstep.RunInputOf(inputVerbKey, raw) if err != nil { - return machineEnd{couldNot: node + ": " + err.Error()} + end(node + ": " + err.Error()) + return + } + if _, _, err := r.runs.change(key, func(c *meshwants.CompositionRun) bool { + if c.Ask != ask || c.Ended != nil { + return false + } + c.Program = &meshwants.CompositionRunProgram{Repository: program.Repository, Commit: program.Commit, Directory: program.Dir, Fingerprint: engineFp} + c.Input = digest + return true + }); err != nil { + end(node + ": the run's program and input are not written: " + err.Error()) + return } a := buildstep.StepAsk{ID: ask, Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: runAsker, At: r.now().UTC(), Run: &buildstep.RunJob{Program: program, Heads: []buildstep.RunHead{}, Input: ref, Reader: "the composition of " + node + " by the node-engine at " + program.Commit}} - ended := r.ends.expect(ask) - defer r.ends.forget(ask) if err := r.put(ctx, a); err != nil { - return machineEnd{couldNot: node + ": its run ask " + ask + " is not put: " + err.Error()} + end(node + ": its run ask " + ask + " is not put: " + err.Error()) } - var e buildstep.StepEnded - select { - case <-ctx.Done(): - return machineEnd{couldNot: node + ": the step stopped before its run ask " + ask + " ended"} - case e = <-ended: - } - if e.Outcome != buildstep.OutcomePassed { - return machineEnd{couldNot: fmt.Sprintf("%s: its run ask %s ended %s on %s: %s", node, ask, e.Outcome, e.Node, e.Said)} - } - answer, err := e.ReadOutput(buildstep.AskGo, r.ask) // the program's last line of standard output (A-254) - if err != nil { - return machineEnd{couldNot: node + ": " + err.Error()} - } - return verdictOf(node, answer) } -// verdictOf reads the composer's verdict of one machine: not composing is failed, naming why; not judged is could not run, naming why (A-467). -func verdictOf(node string, answer []byte) machineEnd { +// judgeRun judges one machine of a step from the run it read: could not run with the run's words; else from the machine's +// composer verdict: not judged is could not run, not composing is failed, naming why (A-467). +func judgeRun(run meshwants.CompositionRun) machineEnd { + node := run.Machine + if run.Ended == nil { + return machineEnd{couldNot: node + ": the step stopped before its composition run " + runKeyOf(run.Candidate, node) + " ended"} + } + if run.Outcome != meshwants.CompositionRunOutcomePassed { + return machineEnd{couldNot: run.Said} + } + m, err := machineOf(run.Verdict) + if err != nil { + return machineEnd{couldNot: node + ": the run's verdict does not read: " + err.Error()} + } + switch { + case m.NotJudged != "": // a step never passes on a machine it did not judge (A-467) + return machineEnd{couldNot: node + " was not judged: " + m.NotJudged} + case !m.Composes: + return machineEnd{judged: true, failed: node + " does not compose: " + strings.Join(appendSome(append(slices.Clone(m.Problems), m.Refusals...), m.Reason), "; ")} + } + return machineEnd{judged: true} +} + +// machineOf is the composer's verdict of one machine as a run keeps it. +func machineOf(v map[string]any) (composer.Machine, error) { + var m composer.Machine + raw, err := json.Marshal(v) + if err == nil { + err = json.Unmarshal(raw, &m) + } + return m, err +} + +// runVerdict reads the composer's answer of one machine: its verdict as the run keeps it, or why it is none. +func runVerdict(node string, answer []byte) (map[string]any, string) { var v composer.Verdict if err := json.Unmarshal(answer, &v); err != nil { - return machineEnd{couldNot: node + ": the composer answered no verdict: " + err.Error()} + return nil, node + ": the composer answered no verdict: " + err.Error() } if v.V != composer.Version { - return machineEnd{couldNot: fmt.Sprintf("%s: the composer answered a verdict of version %d, the assigner reads %d", node, v.V, composer.Version)} + return nil, fmt.Sprintf("%s: the composer answered a verdict of version %d, the assigner reads %d", node, v.V, composer.Version) } for _, c := range v.Cases { if c.Name != changeCase { continue } m, ok := c.Machines[node] - switch { - case !ok: - return machineEnd{couldNot: node + ": the composer's verdict names no such machine"} - case m.NotJudged != "": // a step never passes on a machine it did not judge (A-467) - return machineEnd{couldNot: node + " was not judged: " + m.NotJudged} - case !m.Composes: - return machineEnd{judged: true, failed: node + " does not compose: " + strings.Join(appendSome(append(slices.Clone(m.Problems), m.Refusals...), m.Reason), "; ")} + if !ok { + return nil, node + ": the composer's verdict names no such machine" } - return machineEnd{judged: true} + raw, err := json.Marshal(m) + var out map[string]any + if err == nil { + err = json.Unmarshal(raw, &out) + } + if err != nil { + return nil, node + ": the composer's verdict cannot be kept: " + err.Error() + } + return out, "" } - return machineEnd{couldNot: node + ": the composer's verdict has no case " + changeCase} + return nil, node + ": the composer's verdict has no case " + changeCase } // revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252), once for every step of @@ -470,50 +519,53 @@ func programOfEntry(e meshcatalog.CatalogEntry) (buildstep.RunProgram, error) { return buildstep.RunProgram{Repository: e.Repository, Commit: e.Commit, Dir: dir, Package: composer.Package}, nil } -// programs answers, per machine, the composer its run builds (A-226, A-243, A-252): where the revision's node-engine entry differs -// from the trunk's, that build for every machine; else the build the machine's node-engine runs, as its `running` says, its entry -// read through mesh-catalog's `entry`; a machine whose node-engine's `running` is not heard gets the revision's entry, said (A-451). -func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, error), error) { +// programs answers, per machine, the composer its run builds and the node-engine fingerprint it was chosen from (A-226, A-243, +// A-252, §719): where the revision's node-engine entry differs from the trunk's, that build for every machine; else the build the +// machine's node-engine runs, as its `running` says, its entry read through mesh-catalog's `entry`; a machine whose node-engine's +// `running` is not heard gets the revision's entry, said (A-451). +func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, string, error), error) { engine, ok := nodeEngineEntry(half.Entries) if !ok { return nil, fmt.Errorf("the catalog revision %d holds no module claiming the seat %s", half.Revision.Revision, ne.Seat) } if trunk, ok := r.trunk(); !ok || func() bool { t, ok := nodeEngineEntry(trunk.Entries); return !ok || t.Fp != engine.Fp }() { p, err := programOfEntry(engine) - return func(string) (buildstep.RunProgram, error) { return p, err }, nil + return func(string) (buildstep.RunProgram, string, error) { return p, engine.Fp, err }, nil } var mu sync.Mutex byFp := map[string]buildstep.RunProgram{} - return func(node string) (buildstep.RunProgram, error) { + return func(node string) (buildstep.RunProgram, string, error) { running, ok := r.running(node, engine.Module) if !ok || running.Fingerprint == "" { r.logf("%s: no build of %s is heard running there, so its composer is the catalog revision's entry %s (A-451)", node, engine.Module, engine.Fp) - return programOfEntry(engine) + p, err := programOfEntry(engine) + return p, engine.Fp, err } + fp := running.Fingerprint mu.Lock() - p, ok := byFp[running.Fingerprint] + p, ok := byFp[fp] mu.Unlock() if ok { - return p, nil + return p, fp, nil } - what := "the entry of the node-engine's build " + running.Fingerprint + " is not read" - body, said, err := r.callRetry(ctx, entryVerbKey, meshcatalog.EntryInput{Fingerprint: running.Fingerprint}) + what := "the entry of the node-engine's build " + fp + " is not read" + body, said, err := r.callRetry(ctx, entryVerbKey, meshcatalog.EntryInput{Fingerprint: fp}) if err != nil || body == nil { - return buildstep.RunProgram{}, unreadErr(what, err, said) + return buildstep.RunProgram{}, fp, unreadErr(what, err, said) } var a meshcatalog.EntryAnswer if err := json.Unmarshal(body, &a); err != nil { - return buildstep.RunProgram{}, fmt.Errorf("%s: %w", what, err) + return buildstep.RunProgram{}, fp, fmt.Errorf("%s: %w", what, err) } p, err = programOfEntry(a.Entry) if err != nil { - return p, err + return p, fp, err } mu.Lock() - byFp[running.Fingerprint] = p + byFp[fp] = p mu.Unlock() - return p, nil + return p, fp, nil }, nil } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go index d64e621e..b631594b 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go @@ -30,6 +30,8 @@ type runStand struct { requests map[string]composer.Request // by node entries []string // the fingerprints `entry` was asked r *compositionRun + runs *compositionRuns + puts int // the run asks put // noDefinitions: the revision names no seat definition (A-234's stand-in); trunkSeats: the trunk's seat list as (seat, digest) // pairs; unjudged: the machine the composer does not judge; listFault: the trunk's seat list cannot be read. noDefinitions bool @@ -99,10 +101,10 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) * away := &awayNodes{} away.set("shanks", true) keeper := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now) - ends := &runEnds{} - s.r = &compositionRun{ + s.runs = newCompositionRuns(newRunsMem(), defaultRunSettings, time.Now, t.Logf) + s.r = &compositionRun{runs: s.runs, inputs: &compositionInputs{machines: func(string, string) []string { return []string{"ace", "g14", "shanks"} }, callFile: engines, bound: time.Second}, - keeper: keeper, reads: &compositionReads{}, call: call, ends: ends, away: away, now: time.Now, logf: t.Logf, + keeper: keeper, reads: &compositionReads{}, call: call, away: away, now: time.Now, logf: t.Logf, trunk: func() (mcc.Half, bool) { return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{"engine": engine(trunkEngine, commitE1)}}, true }, @@ -130,7 +132,15 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) * m = composer.Machine{Reason: "two modules own one path", Problems: []string{"/etc/x is m's and bad's"}} } v, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: changeCase, Machines: map[string]composer.Machine{node: m}}}}) - go ends.heard(buildstep.StepEnded{Ask: a.ID, Asker: runAsker, Node: "builder1", Outcome: buildstep.OutcomePassed, Output: string(v)}) + s.mu.Lock() + s.puts++ + s.mu.Unlock() + go func() { + if err := s.runs.heardEnd(buildstep.StepEnded{Ask: a.ID, Asker: runAsker, Node: "builder1", Outcome: buildstep.OutcomePassed, Output: string(v)}, + func(e buildstep.StepEnded) ([]byte, error) { return []byte(e.Output), nil }); err != nil { + t.Error(err) + } + }() return nil }, } @@ -329,3 +339,69 @@ func TestTheCompositionStepReadsTheRevisionOnceAndAgainOnATimeout(t *testing.T) }) } } + +// Design 47 G §718, §719, §720: three composition steps of one revision on two machines start two runs, one request and one run +// ask per machine, and each step reads both verdicts; the run records its program with the node-engine fingerprint it was chosen +// from and the digest of its kept request, which names no module of the asking pipeline; a step of another candidate starts its +// own runs, and a run that could not run is never read again. +func TestTheStepsOfOneCandidateReadOneRunPerMachine(t *testing.T) { + s := newRunStand(t, "src2:e1", "src2:e1", "d1") + s.r.inputs.machines = func(string, string) []string { return []string{"ace", "g14"} } + var wg sync.WaitGroup + outs := make([]stepOutcome, 3) + for i, m := range []string{"m", "m2", "m3"} { + wg.Add(1) + go func() { + defer wg.Done() + outs[i] = s.r.run(context.Background(), "ask-"+m, meshwants.StepRun{Pipeline: "main.7." + m, Module: m, Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}) + }() + } + wg.Wait() + for i, o := range outs { + if o.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(o.Said, "g14 does not compose") { + t.Errorf("step %d came to %+v, want both machines read", i, o) + } + } + if s.puts != 2 { + t.Fatalf("%d run asks put, want one per machine", s.puts) + } + ace, _, _, _ := s.runs.read(runKeyOf("main.7", "ace")) + g14, _, _, _ := s.runs.read(runKeyOf("main.7", "g14")) + if len(ace.Readers) != 3 || len(g14.Readers) != 3 || ace.Program == nil || ace.Program.Fingerprint != "src2:e1" || ace.Program.Commit != commitE1 || + g14.Program == nil || g14.Program.Fingerprint != "src2:e2" || ace.Input == "" || ace.Input != digestOf(mustJSON(t, s.requests["ace"])) || + ace.Outcome != meshwants.CompositionRunOutcomePassed || g14.Outcome != meshwants.CompositionRunOutcomePassed || g14.Verdict["composes"] != false { + t.Errorf("the runs: ace %+v, g14 %+v", ace, g14) + } + if req := s.requests["ace"]; strings.Contains(string(mustJSON(t, req)), "main.7.m") || string(req.Input) != `{"node":"ace"}` { + t.Errorf("the request names the asking pipeline or not the node's input: %s", mustJSON(t, req)) + } + // Another candidate (a run on request) starts its own runs. + if o := s.r.run(context.Background(), "ask-r", meshwants.StepRun{Pipeline: "run-r1.m", Module: "m", Branch: "main", Revision: 7, Run: "r1", Step: meshwants.StepRunStepComposition}); o.Outcome != meshwants.StepEndedOutcomeFailed || s.puts != 4 { + t.Errorf("a step of another candidate came to %+v with %d asks, want its own two runs", o, s.puts) + } + // A run whose ask could not be put could not run; the next step of the candidate starts it again. + f := newRunStand(t, "src2:e1", "src2:e1", "d1") + f.r.inputs.machines = func(string, string) []string { return []string{"ace"} } + put, refused := f.r.put, 0 + f.r.put = func(ctx context.Context, a buildstep.StepAsk) error { + if refused == 0 { + refused++ + return errors.New("no stream") + } + return put(ctx, a) + } + first := f.r.run(context.Background(), "ask-1", meshwants.StepRun{Pipeline: "main.7.m", Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}) + second := f.r.run(context.Background(), "ask-2", meshwants.StepRun{Pipeline: "main.7.m2", Module: "m2", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}) + if first.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(first.Said, "is not put") || second.Outcome != meshwants.StepEndedOutcomePassed || f.puts != 1 { + t.Errorf("after a run that could not run: %+v then %+v, %d asks", first, second, f.puts) + } +} + +func mustJSON(t *testing.T, v any) []byte { + t.Helper() + raw, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return raw +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go index c235ed5a..f097eace 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go @@ -15,6 +15,7 @@ import ( "github.com/nats-io/nats.go/jetstream" meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/keys" "mesh-delivery-assigner/internal/buckets" @@ -441,3 +442,107 @@ func watchRuns(ctx context.Context, kv jetstream.KeyValue, r *compositionRuns, e }() return nil } + +// end ends the run of ask in its key with an outcome, its words and the machine's verdict, by compare-and-set; it answers +// false when the key holds another run or the run has ended already. +func (r *compositionRuns) end(key, ask, outcome, said string, verdict map[string]any) (bool, error) { + ended := false + _, _, err := r.change(key, func(c *meshwants.CompositionRun) bool { + if c.Ask != ask || c.Ended != nil { + return false + } + at := r.now().UTC() + c.Ended, c.Outcome, c.Said, c.Verdict = &at, outcome, said, verdict + ended = true + return true + }) + return ended, err +} + +// keyOfAsk is the key of the run whose ask is ask, or none. +func (r *compositionRuns) keyOfAsk(ask string) (string, meshwants.CompositionRun, error) { + all, err := r.kv.keys() + if err != nil { + return "", meshwants.CompositionRun{}, err + } + for _, key := range all { + if !isRunKey(key) { + continue + } + run, _, ok, err := r.read(key) + if err == nil && ok && run.Ask == ask { + return key, run, nil + } + } + return "", meshwants.CompositionRun{}, nil +} + +// heardEnd ends the run of the ask a build seat's step-ended names (§719, A-240): a run ask that ended other than passed +// could not run; a passed one's answer, read with read, is the machine's verdict, or why it is none. An end of an ask no run +// names is another asker's, or of a run gone, and changes nothing. +func (r *compositionRuns) heardEnd(e buildstep.StepEnded, read func(buildstep.StepEnded) ([]byte, error)) error { + if e.Asker != runAsker || e.Ask == "" { + return nil + } + key, run, err := r.keyOfAsk(e.Ask) + if err != nil || key == "" { + return err + } + if run.Ended != nil { + return nil + } + node := run.Machine + outcome, said := meshwants.CompositionRunOutcomeCouldNotRun, "" + var verdict map[string]any + if e.Outcome != buildstep.OutcomePassed { + said = fmt.Sprintf("%s: its run ask %s ended %s on %s: %s", node, e.Ask, e.Outcome, e.Node, e.Said) + } else if answer, err := read(e); err != nil { // the program's last line of standard output (A-254) + said = node + ": " + err.Error() + } else if verdict, said = runVerdict(node, answer); said == "" { + outcome = meshwants.CompositionRunOutcomePassed + } + _, err = r.end(key, e.Ask, outcome, said, verdict) + return err +} + +// await waits until each run of keys has ended, through the state's changes heard, or ctx ends: it answers each run as it +// stands then; a key gone could not run. +func (r *compositionRuns) await(ctx context.Context, keys []string) map[string]meshwants.CompositionRun { + out := map[string]meshwants.CompositionRun{} + for { + wake := r.wake() + open := false + for _, key := range keys { + run, _, ok, err := r.read(key) + switch { + case err != nil: + open = true + continue + case !ok: + at := r.now().UTC() + candidate, machine, _ := strings.Cut(key, ".") + run = meshwants.CompositionRun{Candidate: unescape(candidate), Machine: unescape(machine), Ended: &at, + Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: unescape(machine) + ": its composition run " + key + " is gone"} + case run.Ended == nil: + open = true + } + out[key] = run + } + if !open { + return out + } + select { + case <-ctx.Done(): + return out + case <-wake: + } + } +} + +// unescape is a key's part as a name again; a part that does not read stays as it is. +func unescape(part string) string { + if name, err := keys.UnescapeName(part); err == nil { + return name + } + return part +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs.go index ba5f7bc4..fae279b4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs.go @@ -555,8 +555,9 @@ func inputsDrop(k *inputKeeper, ask func(string, any) (json.RawMessage, error)) } // heardStepEnded deletes the input kept for the ask a build seat's step-ended names (design 47 G §354), and hands the end to the -// composition run that waits for that ask (A-240); one whose input cannot be deleted is given again by the bus. -func heardStepEnded(k *inputKeeper, ends *runEnds, m jetstream.Msg, logf func(string, ...any)) { +// composition run of that ask (A-240, §719), which writes it in the run's key; one whose input cannot be deleted, or whose run's +// end cannot be written, is given again by the bus. +func heardStepEnded(k *inputKeeper, runEnded func(buildstep.StepEnded) error, m jetstream.Msg, logf func(string, ...any)) { var e buildstep.StepEnded if err := json.Unmarshal(m.Data(), &e); err != nil { logf("%s holds what is no step-ended, left: %v", m.Subject(), err) @@ -568,8 +569,12 @@ func heardStepEnded(k *inputKeeper, ends *runEnds, m jetstream.Msg, logf func(st _ = m.Nak() return } - if ends != nil { - ends.heard(e) + if runEnded != nil { + if err := runEnded(e); err != nil { + logf("the composition run of the ask %s is not ended, so the bus gives its step-ended again: %v", e.Ask, err) + _ = m.Nak() + return + } } _ = m.Ack() } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 6f63dcc4..3dabcb87 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -11,6 +11,7 @@ import ( "fmt" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/buscalls" "io" "log" @@ -527,9 +528,27 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, // The build seat's step-ended of an ask deletes the input kept for it (design 47 G §354) and ends the composition run that waits // for it (A-240); the watchman's machine-away and machine-back say which machines a composition step does not compose (§64, // §389). Registered before the hub starts. - ends, away := &runEnds{}, &awayNodes{} + away := &awayNodes{} + // The composition runs (design 47 G §718, §719), watched in their own state; a build seat's step-ended of a run's ask ends + // the run in its key. + var runs *compositionRuns + if kv := kvs[buckets.CompositionRuns()]; kv != nil { + set, err := runSettingsOf(getenv) + if err != nil { + return err + } + runs = newCompositionRuns(busRuns{ctx, kv}, set, time.Now, logger.Printf) + if err := watchRuns(ctx, kv, runs, nil, nil); err != nil { + return fmt.Errorf("the state %s is not watched: %w", buckets.CompositionRunsState, err) + } + } if inputs != nil { - if err := hub.Handle(nodebuildgo.SubjectStepEndedAll, func(m jetstream.Msg) { heardStepEnded(inputs, ends, m, logger.Printf) }); err != nil { + var runEnded func(buildstep.StepEnded) error + if runs != nil && bus != nil { + read := func(e buildstep.StepEnded) ([]byte, error) { return e.ReadOutput(buildstep.AskGo, bus.CallFile) } + runEnded = func(e buildstep.StepEnded) error { return runs.heardEnd(e, read) } + } + if err := hub.Handle(nodebuildgo.SubjectStepEndedAll, func(m jetstream.Msg) { heardStepEnded(inputs, runEnded, m, logger.Printf) }); err != nil { return err } } @@ -683,17 +702,9 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, p.acts = newPutActs(bus, wake, logger.Printf) } // The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine. - if bus != nil && js != nil && inputs != nil && p.publishWants != nil && kvs[buckets.CompositionRuns()] != nil { - set, err := runSettingsOf(getenv) - if err != nil { - return err - } - runs := newCompositionRuns(busRuns{ctx, kvs[buckets.CompositionRuns()]}, set, time.Now, logger.Printf) - if err := watchRuns(ctx, kvs[buckets.CompositionRuns()], runs, nil, nil); err != nil { - return fmt.Errorf("the state %s is not watched: %w", buckets.CompositionRunsState, err) - } + if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil { run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, - trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ends: ends, ask: bus.CallFile, now: time.Now, logf: logger.Printf} + trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf} steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, now: time.Now, logf: logger.Printf, asker: stepAskerOf(seatlist.Definition(gate.Reader, meshwants.Seat))} if err := serveCompositionSteps(ctx, nc, steps); err != nil {