G-24: one composition run per candidate and machine, one test run per level, every reading step timed #244

Merged
mesh-tools merged 49 commits from fix/47-G-shared-runs into main 2026-11-17 12:15:36 +00:00
7 changed files with 439 additions and 189 deletions
Showing only changes of commit afa95fd0df - Show all commits
@@ -10,7 +10,6 @@ import (
"errors"
"fmt"
"slices"
"sync"
"time"
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
@@ -18,8 +17,8 @@ import (
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// The inputs of a composition step (design 47 G §354, A-243, A-252): one request per machine, each carrying that machine's input,
// asked of the machine's node-engine by the seat's verb compose-candidate.
// The inputs of a composition run (design 47 G §354, §718, A-243, A-252): one request per candidate and machine, carrying that
// machine's input, asked of the machine's node-engine by the seat's verb compose-candidate when the run starts.
// composeCandidateKey is the seat verb's call key, asked of one node.
const composeCandidateKey = "seat:" + ne.Seat + "." + ne.VerbComposeCandidate
@@ -80,37 +79,14 @@ type compositionInputs struct {
bound time.Duration
}
// gather asks each machine of the step's module and branch for its input with the step's candidate key, and answers every machine in
// name order: one away (away) is not asked; a node that is silent, unready or answers no input has Why set; no machine at all is an
// error. The run keeps each request built from an input under its run ask (compositionRun.machine).
func (c *compositionInputs) gather(ctx context.Context, run meshwants.StepRun, away func(node string) bool) ([]nodeInput, error) {
key, err := candidateKeyOf(run)
if err != nil {
return nil, err
}
nodes := c.machines(run.Module, run.Branch)
if len(nodes) == 0 {
return nil, fmt.Errorf("no node places %s on the branch %q, so the composition step has no machine to compose", run.Module, run.Branch)
}
// of asks the machine node for its input with the candidate's key (§718): asked only by the composition step that starts the
// machine's composition run, once per candidate and machine. A node that is silent, unready or answers no input has Why set.
func (c *compositionInputs) of(ctx context.Context, node, candidate string) nodeInput {
bound := c.bound
if bound <= 0 {
bound = nec.AnswerBound
}
out := make([]nodeInput, len(nodes))
var wg sync.WaitGroup
for i, node := range nodes {
if away != nil && away(node) {
out[i] = nodeInput{Node: node, Away: true}
continue
}
wg.Add(1)
go func() {
defer wg.Done()
out[i] = c.one(ctx, bound, node, key)
}()
}
wg.Wait()
return out, nil
return c.one(ctx, bound, node, candidate)
}
// one asks one node for its input.
@@ -68,7 +68,7 @@ func TestTheMachinesOfAStep(t *testing.T) {
// §354, A-243, A-252, §64: each machine is asked compose-candidate with the step's key and input true, its numbers as written; a
// silent or unready node says why, and a machine away is not asked.
func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
// gather asks the machines side by side, so the record of what each was asked is guarded.
// the machines are asked side by side, so the record of what each was asked is guarded.
var mu sync.Mutex
asked := map[string]string{}
call := func(key string, body any) (json.RawMessage, error) {
@@ -86,7 +86,7 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
return json.RawMessage(`{"candidate":"run-r1","node":"` + node + `","verdict":"composed","input":{"n":12345678901234567890,"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
}
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14", "novox", "shanks"} }, callFile: call, bound: time.Second}
got, err := c.gather(context.Background(), meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" })
got, err := gatherAll(c, meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" })
if err != nil || len(got) != 4 {
t.Fatalf("%+v %v", got, err)
}
@@ -105,7 +105,7 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
t.Errorf("the machine away is asked or not said away: %+v", got[3])
}
none := &compositionInputs{machines: func(string, string) []string { return nil }, callFile: call}
if _, err := none.gather(context.Background(), meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil {
if _, err := gatherAll(none, meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil {
t.Error("a step with no machine is gathered")
}
}
@@ -124,7 +124,7 @@ func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) {
return json.RawMessage(answer + `,"file":"` + file + `"}`), nil
}
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14"} }, callFile: call, bound: time.Second}
got, err := c.gather(context.Background(), meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(string) bool { return false })
got, err := gatherAll(c, meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(string) bool { return false })
if err != nil || len(got) != 2 {
t.Fatalf("%+v %v", got, err)
}
@@ -135,3 +135,28 @@ func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) {
t.Errorf("g14: %+v", got[1])
}
}
// gatherAll asks each machine of a step for its input as each run's start does (compositionInputs.of), side by side; a machine away
// is not asked; no machine is an error.
func gatherAll(c *compositionInputs, run meshwants.StepRun, away func(string) bool) ([]nodeInput, error) {
key, err := candidateKeyOf(run)
if err != nil {
return nil, err
}
nodes := c.machines(run.Module, run.Branch)
if len(nodes) == 0 {
return nil, errors.New("no machine")
}
out := make([]nodeInput, len(nodes))
var wg sync.WaitGroup
for i, node := range nodes {
if away != nil && away(node) {
out[i] = nodeInput{Node: node, Away: true}
continue
}
wg.Add(1)
go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key) }()
}
wg.Wait()
return out, nil
}
@@ -21,12 +21,14 @@ import (
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The run of a composition step (design 47 G §354, §63, §64; A-226, A-238, A-240, A-243, A-252, A-254, A-261): one
// composer.Request per machine, depth compose, the one machine and one case `change`, its manifests, verdicts and seats from the
// step's catalog revision read through mesh-catalog's `revision`; one run ask per machine of the seat node-build-go, naming no
// head, its program the node-engine build's catalog entry with the composer's package; the run's end heard on the build seat's
// step-ended of that ask, its answer (a composer.Verdict) read inline from `output`. A machine the watchman said away
// (machine-away) is not composed and passes with a warning (§64).
// The run of a composition step (design 47 G §354, §63, §64, §718; A-226, A-238, A-240, A-252, A-254, A-261): the step reads one
// composition run per machine of its candidate (compositionruns.go), and the step that starts a machine's run composes it: one
// compose-candidate ask of the node, one composer.Request, depth compose, the one machine and one case `change`, its manifests,
// verdicts and seats from the step's catalog revision read through mesh-catalog's `revision`, kept once in `inputs`; one run ask
// of the seat node-build-go, naming no head, its program the node-engine build's catalog entry with the composer's package,
// chosen once and recorded in the run. The run's end is heard on the build seat's step-ended of that ask, its answer (a
// composer.Verdict) read from `output`, and written in the run's key, whose change every reading step hears. A machine the
// watchman said away (machine-away) is not composed and passes with a warning (§64).
const (
// runAsker is the assigner's name as an asker of a build seat, its seat's own name: the last token of the subject its run
@@ -41,42 +43,6 @@ const (
seatDefinitionsKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbSeatDefinitions
)
// runEnds hands each build seat's step-ended of an ask to the composition run that waits for it (A-240), by the ask's id.
type runEnds struct {
mu sync.Mutex
waits map[string]chan buildstep.StepEnded
}
// expect is the end of the ask id, once heard; called before the ask is put, so no end is missed.
func (r *runEnds) expect(id string) <-chan buildstep.StepEnded {
r.mu.Lock()
defer r.mu.Unlock()
if r.waits == nil {
r.waits = map[string]chan buildstep.StepEnded{}
}
ch := make(chan buildstep.StepEnded, 1)
r.waits[id] = ch
return ch
}
// forget stops waiting for the ask id.
func (r *runEnds) forget(id string) {
r.mu.Lock()
delete(r.waits, id)
r.mu.Unlock()
}
// heard hands an end to the run that waits for its ask; one nobody waits for is another asker's, left.
func (r *runEnds) heard(e buildstep.StepEnded) {
r.mu.Lock()
ch := r.waits[e.Ask]
delete(r.waits, e.Ask)
r.mu.Unlock()
if ch != nil {
ch <- e
}
}
// awayNodes are the machines the watchman said away (machine-away of the seat mesh-watchdogs, §389) and not back since.
type awayNodes struct {
mu sync.Mutex
@@ -121,8 +87,7 @@ type compositionRun struct {
running func(node, module string) (ne.Running, bool)
away *awayNodes
// put puts a run ask on the build seat's check queue under the assigner's name.
put func(ctx context.Context, a buildstep.StepAsk) error
ends *runEnds
put func(ctx context.Context, a buildstep.StepAsk) error
// reads shares the catalog reads among steps (compositionreads.go); sleep waits between two tries of a read (nil: a timer).
reads *compositionReads
sleep func(ctx context.Context, d time.Duration) error
@@ -139,41 +104,49 @@ type machineEnd struct {
judged bool
}
// run runs one composition step and answers what it came to: failed when a machine does not compose, could not run when a machine
// could not be composed or was not judged, else passed with a warning per machine away.
func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRun) stepOutcome {
half, err := r.revision(ctx, run)
if err != nil {
return couldNotRun("the step's catalog revision is not read: " + err.Error())
}
seats, err := r.seats(ctx, half.Revision.Definitions)
// run runs one composition step and answers what it came to: per machine not away it joins or starts the candidate's
// composition run (§718), waits for each run's end in its key, and judges each machine from it; failed when a machine does not
// compose, could not run when a machine could not be composed or was not judged, else passed with a warning per machine away.
func (r *compositionRun) run(ctx context.Context, id string, step meshwants.StepRun) stepOutcome {
candidate, err := candidateKeyOf(step)
if err != nil {
return couldNotRun(err.Error())
}
manifests, verdicts, err := caseOf(half)
if err != nil {
return couldNotRun(err.Error())
nodes := r.inputs.machines(step.Module, step.Branch)
if len(nodes) == 0 {
return couldNotRun(fmt.Sprintf("no node places %s on the branch %q, so the composition step has no machine to compose", step.Module, step.Branch))
}
programOf, err := r.programs(ctx, half)
if err != nil {
return couldNotRun(err.Error())
}
inputs, err := r.inputs.gather(ctx, run, r.away.is)
if err != nil {
return couldNotRun(err.Error())
reader := readerOf(id, step)
prep := &runPrep{r: r, step: step}
var ends []machineEnd
var keys []string
for _, node := range nodes {
if r.away.is(node) {
ends = append(ends, machineEnd{warning: node + " is away (the watchman's machine-away), so it is not composed"})
continue
}
j, err := r.runs.join(reader, candidate, node, buildstep.NewID())
if err != nil {
ends = append(ends, machineEnd{couldNot: node + ": " + err.Error()})
continue
}
if j.started {
go r.start(ctx, j.key, j.run, prep)
}
keys = append(keys, j.key)
}
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
ends := make([]machineEnd, len(inputs))
var wg sync.WaitGroup
for i, in := range inputs {
wg.Add(1)
go func() {
defer wg.Done()
ends[i] = r.machine(ctx, in, buildstep.NewID(), programOf, manifests, verdicts, seats)
}()
}
wg.Wait()
runs := r.runs.await(ctx, keys)
setPhase(ctx, meshwants.StepProgressEventPhaseReport)
for _, key := range keys {
ends = append(ends, judgeRun(runs[key]))
}
return outcomeOf(ends)
}
// outcomeOf is a step's outcome from its machines: failed before could not run before passed (§354); a step that judged no
// machine never passes (A-467a), each machine not judged named with why.
func outcomeOf(ends []machineEnd) stepOutcome {
var out stepOutcome
var failed, couldNot []string
judged := 0
@@ -190,7 +163,7 @@ func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRu
out.Outcome, out.Said = meshwants.StepEndedOutcomeFailed, strings.Join(failed, "; ")
case len(couldNot) > 0:
out.Outcome, out.Said = meshwants.StepEndedOutcomeCouldNotRun, strings.Join(couldNot, "; ")
case judged == 0: // a step that judged no machine never passes (A-467a): each machine not judged is named with why
case judged == 0:
said := "no machine was judged"
if len(out.Warnings) == 0 {
said += ": the step has no machine"
@@ -215,82 +188,158 @@ func couldNotRun(said string) stepOutcome {
return stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: said}
}
// machine composes one machine: its request kept under its ask, the ask put, its end heard and its verdict read.
func (r *compositionRun) machine(ctx context.Context, in nodeInput, ask string, programOf func(node string) (buildstep.RunProgram, error),
manifests map[string]composer.Manifest, verdicts map[string]json.RawMessage, seats []json.RawMessage) machineEnd {
node := in.Node
switch {
case in.Away:
return machineEnd{warning: node + " is away (the watchman's machine-away), so it is not composed"}
case in.Why != "":
return machineEnd{couldNot: node + " gave no composition input: " + in.Why}
// runPrep is what the step that starts a run reads of its catalog revision, once for every machine it starts: the seats, the
// case's manifests and verdicts, and the composer program per machine.
type runPrep struct {
r *compositionRun
step meshwants.StepRun
once sync.Once
// read once
seats []json.RawMessage
manifests map[string]composer.Manifest
verdicts map[string]json.RawMessage
programOf func(node string) (buildstep.RunProgram, string, error)
err error
}
func (p *runPrep) get(ctx context.Context) error {
p.once.Do(func() {
half, err := p.r.revision(ctx, p.step)
if err != nil {
p.err = errors.New("the step's catalog revision is not read: " + err.Error())
return
}
if p.seats, p.err = p.r.seats(ctx, half.Revision.Definitions); p.err != nil {
return
}
if p.manifests, p.verdicts, p.err = caseOf(half); p.err != nil {
return
}
p.programOf, p.err = p.r.programs(ctx, half)
})
return p.err
}
// start composes the machine of the run it started (§718): the node's input of this moment, its request kept once in `inputs`,
// the composer program chosen and recorded with the request's digest, and one run ask put. A fault ends the run could not run.
func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep) {
node, ask := run.Machine, run.Ask
end := func(said string) {
if _, err := r.runs.end(key, ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil); err != nil {
r.logf("the composition run %s is not ended: %v", key, err)
}
}
program, err := programOf(node)
if err := prep.get(ctx); err != nil {
end(node + ": " + err.Error())
return
}
in := r.inputs.of(ctx, node, run.Candidate)
if in.Why != "" {
end(node + " gave no composition input: " + in.Why)
return
}
program, engineFp, err := prep.programOf(node)
if err != nil {
return machineEnd{couldNot: node + ": " + err.Error()}
end(node + ": " + err.Error())
return
}
req := composer.Request{V: composer.Version, Input: in.Body, Depth: composer.DepthCompose, Machines: []string{node},
Cases: []composer.Case{{Name: changeCase, Manifests: manifests, Verdicts: verdicts}}, Seats: seats}
Cases: []composer.Case{{Name: changeCase, Manifests: prep.manifests, Verdicts: prep.verdicts}}, Seats: prep.seats}
raw, err := json.Marshal(req)
if err != nil {
return machineEnd{couldNot: node + ": its request cannot be encoded: " + err.Error()}
end(node + ": its request cannot be encoded: " + err.Error())
return
}
if _, err := r.keeper.keep(ask, raw); err != nil {
digest, err := r.keeper.keep(ask, raw)
if err != nil {
r.logf("the composition request of %s is refused: %v", node, err)
return machineEnd{couldNot: node + ": its request is not kept: " + err.Error()}
end(node + ": its request is not kept: " + err.Error())
return
}
ref, err := buildstep.RunInputOf(inputVerbKey, raw)
if err != nil {
return machineEnd{couldNot: node + ": " + err.Error()}
end(node + ": " + err.Error())
return
}
if _, _, err := r.runs.change(key, func(c *meshwants.CompositionRun) bool {
if c.Ask != ask || c.Ended != nil {
return false
}
c.Program = &meshwants.CompositionRunProgram{Repository: program.Repository, Commit: program.Commit, Directory: program.Dir, Fingerprint: engineFp}
c.Input = digest
return true
}); err != nil {
end(node + ": the run's program and input are not written: " + err.Error())
return
}
a := buildstep.StepAsk{ID: ask, Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: runAsker, At: r.now().UTC(),
Run: &buildstep.RunJob{Program: program, Heads: []buildstep.RunHead{}, Input: ref, Reader: "the composition of " + node + " by the node-engine at " + program.Commit}}
ended := r.ends.expect(ask)
defer r.ends.forget(ask)
if err := r.put(ctx, a); err != nil {
return machineEnd{couldNot: node + ": its run ask " + ask + " is not put: " + err.Error()}
end(node + ": its run ask " + ask + " is not put: " + err.Error())
}
var e buildstep.StepEnded
select {
case <-ctx.Done():
return machineEnd{couldNot: node + ": the step stopped before its run ask " + ask + " ended"}
case e = <-ended:
}
if e.Outcome != buildstep.OutcomePassed {
return machineEnd{couldNot: fmt.Sprintf("%s: its run ask %s ended %s on %s: %s", node, ask, e.Outcome, e.Node, e.Said)}
}
answer, err := e.ReadOutput(buildstep.AskGo, r.ask) // the program's last line of standard output (A-254)
if err != nil {
return machineEnd{couldNot: node + ": " + err.Error()}
}
return verdictOf(node, answer)
}
// verdictOf reads the composer's verdict of one machine: not composing is failed, naming why; not judged is could not run, naming why (A-467).
func verdictOf(node string, answer []byte) machineEnd {
// judgeRun judges one machine of a step from the run it read: could not run with the run's words; else from the machine's
// composer verdict: not judged is could not run, not composing is failed, naming why (A-467).
func judgeRun(run meshwants.CompositionRun) machineEnd {
node := run.Machine
if run.Ended == nil {
return machineEnd{couldNot: node + ": the step stopped before its composition run " + runKeyOf(run.Candidate, node) + " ended"}
}
if run.Outcome != meshwants.CompositionRunOutcomePassed {
return machineEnd{couldNot: run.Said}
}
m, err := machineOf(run.Verdict)
if err != nil {
return machineEnd{couldNot: node + ": the run's verdict does not read: " + err.Error()}
}
switch {
case m.NotJudged != "": // a step never passes on a machine it did not judge (A-467)
return machineEnd{couldNot: node + " was not judged: " + m.NotJudged}
case !m.Composes:
return machineEnd{judged: true, failed: node + " does not compose: " + strings.Join(appendSome(append(slices.Clone(m.Problems), m.Refusals...), m.Reason), "; ")}
}
return machineEnd{judged: true}
}
// machineOf is the composer's verdict of one machine as a run keeps it.
func machineOf(v map[string]any) (composer.Machine, error) {
var m composer.Machine
raw, err := json.Marshal(v)
if err == nil {
err = json.Unmarshal(raw, &m)
}
return m, err
}
// runVerdict reads the composer's answer of one machine: its verdict as the run keeps it, or why it is none.
func runVerdict(node string, answer []byte) (map[string]any, string) {
var v composer.Verdict
if err := json.Unmarshal(answer, &v); err != nil {
return machineEnd{couldNot: node + ": the composer answered no verdict: " + err.Error()}
return nil, node + ": the composer answered no verdict: " + err.Error()
}
if v.V != composer.Version {
return machineEnd{couldNot: fmt.Sprintf("%s: the composer answered a verdict of version %d, the assigner reads %d", node, v.V, composer.Version)}
return nil, fmt.Sprintf("%s: the composer answered a verdict of version %d, the assigner reads %d", node, v.V, composer.Version)
}
for _, c := range v.Cases {
if c.Name != changeCase {
continue
}
m, ok := c.Machines[node]
switch {
case !ok:
return machineEnd{couldNot: node + ": the composer's verdict names no such machine"}
case m.NotJudged != "": // a step never passes on a machine it did not judge (A-467)
return machineEnd{couldNot: node + " was not judged: " + m.NotJudged}
case !m.Composes:
return machineEnd{judged: true, failed: node + " does not compose: " + strings.Join(appendSome(append(slices.Clone(m.Problems), m.Refusals...), m.Reason), "; ")}
if !ok {
return nil, node + ": the composer's verdict names no such machine"
}
return machineEnd{judged: true}
raw, err := json.Marshal(m)
var out map[string]any
if err == nil {
err = json.Unmarshal(raw, &out)
}
if err != nil {
return nil, node + ": the composer's verdict cannot be kept: " + err.Error()
}
return out, ""
}
return machineEnd{couldNot: node + ": the composer's verdict has no case " + changeCase}
return nil, node + ": the composer's verdict has no case " + changeCase
}
// revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252), once for every step of
@@ -470,50 +519,53 @@ func programOfEntry(e meshcatalog.CatalogEntry) (buildstep.RunProgram, error) {
return buildstep.RunProgram{Repository: e.Repository, Commit: e.Commit, Dir: dir, Package: composer.Package}, nil
}
// programs answers, per machine, the composer its run builds (A-226, A-243, A-252): where the revision's node-engine entry differs
// from the trunk's, that build for every machine; else the build the machine's node-engine runs, as its `running` says, its entry
// read through mesh-catalog's `entry`; a machine whose node-engine's `running` is not heard gets the revision's entry, said (A-451).
func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, error), error) {
// programs answers, per machine, the composer its run builds and the node-engine fingerprint it was chosen from (A-226, A-243,
// A-252, §719): where the revision's node-engine entry differs from the trunk's, that build for every machine; else the build the
// machine's node-engine runs, as its `running` says, its entry read through mesh-catalog's `entry`; a machine whose node-engine's
// `running` is not heard gets the revision's entry, said (A-451).
func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, string, error), error) {
engine, ok := nodeEngineEntry(half.Entries)
if !ok {
return nil, fmt.Errorf("the catalog revision %d holds no module claiming the seat %s", half.Revision.Revision, ne.Seat)
}
if trunk, ok := r.trunk(); !ok || func() bool { t, ok := nodeEngineEntry(trunk.Entries); return !ok || t.Fp != engine.Fp }() {
p, err := programOfEntry(engine)
return func(string) (buildstep.RunProgram, error) { return p, err }, nil
return func(string) (buildstep.RunProgram, string, error) { return p, engine.Fp, err }, nil
}
var mu sync.Mutex
byFp := map[string]buildstep.RunProgram{}
return func(node string) (buildstep.RunProgram, error) {
return func(node string) (buildstep.RunProgram, string, error) {
running, ok := r.running(node, engine.Module)
if !ok || running.Fingerprint == "" {
r.logf("%s: no build of %s is heard running there, so its composer is the catalog revision's entry %s (A-451)", node,
engine.Module, engine.Fp)
return programOfEntry(engine)
p, err := programOfEntry(engine)
return p, engine.Fp, err
}
fp := running.Fingerprint
mu.Lock()
p, ok := byFp[running.Fingerprint]
p, ok := byFp[fp]
mu.Unlock()
if ok {
return p, nil
return p, fp, nil
}
what := "the entry of the node-engine's build " + running.Fingerprint + " is not read"
body, said, err := r.callRetry(ctx, entryVerbKey, meshcatalog.EntryInput{Fingerprint: running.Fingerprint})
what := "the entry of the node-engine's build " + fp + " is not read"
body, said, err := r.callRetry(ctx, entryVerbKey, meshcatalog.EntryInput{Fingerprint: fp})
if err != nil || body == nil {
return buildstep.RunProgram{}, unreadErr(what, err, said)
return buildstep.RunProgram{}, fp, unreadErr(what, err, said)
}
var a meshcatalog.EntryAnswer
if err := json.Unmarshal(body, &a); err != nil {
return buildstep.RunProgram{}, fmt.Errorf("%s: %w", what, err)
return buildstep.RunProgram{}, fp, fmt.Errorf("%s: %w", what, err)
}
p, err = programOfEntry(a.Entry)
if err != nil {
return p, err
return p, fp, err
}
mu.Lock()
byFp[running.Fingerprint] = p
byFp[fp] = p
mu.Unlock()
return p, nil
return p, fp, nil
}, nil
}
@@ -30,6 +30,8 @@ type runStand struct {
requests map[string]composer.Request // by node
entries []string // the fingerprints `entry` was asked
r *compositionRun
runs *compositionRuns
puts int // the run asks put
// noDefinitions: the revision names no seat definition (A-234's stand-in); trunkSeats: the trunk's seat list as (seat, digest)
// pairs; unjudged: the machine the composer does not judge; listFault: the trunk's seat list cannot be read.
noDefinitions bool
@@ -99,10 +101,10 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
away := &awayNodes{}
away.set("shanks", true)
keeper := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
ends := &runEnds{}
s.r = &compositionRun{
s.runs = newCompositionRuns(newRunsMem(), defaultRunSettings, time.Now, t.Logf)
s.r = &compositionRun{runs: s.runs,
inputs: &compositionInputs{machines: func(string, string) []string { return []string{"ace", "g14", "shanks"} }, callFile: engines, bound: time.Second},
keeper: keeper, reads: &compositionReads{}, call: call, ends: ends, away: away, now: time.Now, logf: t.Logf,
keeper: keeper, reads: &compositionReads{}, call: call, away: away, now: time.Now, logf: t.Logf,
trunk: func() (mcc.Half, bool) {
return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{"engine": engine(trunkEngine, commitE1)}}, true
},
@@ -130,7 +132,15 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
m = composer.Machine{Reason: "two modules own one path", Problems: []string{"/etc/x is m's and bad's"}}
}
v, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: changeCase, Machines: map[string]composer.Machine{node: m}}}})
go ends.heard(buildstep.StepEnded{Ask: a.ID, Asker: runAsker, Node: "builder1", Outcome: buildstep.OutcomePassed, Output: string(v)})
s.mu.Lock()
s.puts++
s.mu.Unlock()
go func() {
if err := s.runs.heardEnd(buildstep.StepEnded{Ask: a.ID, Asker: runAsker, Node: "builder1", Outcome: buildstep.OutcomePassed, Output: string(v)},
func(e buildstep.StepEnded) ([]byte, error) { return []byte(e.Output), nil }); err != nil {
t.Error(err)
}
}()
return nil
},
}
@@ -329,3 +339,69 @@ func TestTheCompositionStepReadsTheRevisionOnceAndAgainOnATimeout(t *testing.T)
})
}
}
// Design 47 G §718, §719, §720: three composition steps of one revision on two machines start two runs, one request and one run
// ask per machine, and each step reads both verdicts; the run records its program with the node-engine fingerprint it was chosen
// from and the digest of its kept request, which names no module of the asking pipeline; a step of another candidate starts its
// own runs, and a run that could not run is never read again.
func TestTheStepsOfOneCandidateReadOneRunPerMachine(t *testing.T) {
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
s.r.inputs.machines = func(string, string) []string { return []string{"ace", "g14"} }
var wg sync.WaitGroup
outs := make([]stepOutcome, 3)
for i, m := range []string{"m", "m2", "m3"} {
wg.Add(1)
go func() {
defer wg.Done()
outs[i] = s.r.run(context.Background(), "ask-"+m, meshwants.StepRun{Pipeline: "main.7." + m, Module: m, Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
}()
}
wg.Wait()
for i, o := range outs {
if o.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(o.Said, "g14 does not compose") {
t.Errorf("step %d came to %+v, want both machines read", i, o)
}
}
if s.puts != 2 {
t.Fatalf("%d run asks put, want one per machine", s.puts)
}
ace, _, _, _ := s.runs.read(runKeyOf("main.7", "ace"))
g14, _, _, _ := s.runs.read(runKeyOf("main.7", "g14"))
if len(ace.Readers) != 3 || len(g14.Readers) != 3 || ace.Program == nil || ace.Program.Fingerprint != "src2:e1" || ace.Program.Commit != commitE1 ||
g14.Program == nil || g14.Program.Fingerprint != "src2:e2" || ace.Input == "" || ace.Input != digestOf(mustJSON(t, s.requests["ace"])) ||
ace.Outcome != meshwants.CompositionRunOutcomePassed || g14.Outcome != meshwants.CompositionRunOutcomePassed || g14.Verdict["composes"] != false {
t.Errorf("the runs: ace %+v, g14 %+v", ace, g14)
}
if req := s.requests["ace"]; strings.Contains(string(mustJSON(t, req)), "main.7.m") || string(req.Input) != `{"node":"ace"}` {
t.Errorf("the request names the asking pipeline or not the node's input: %s", mustJSON(t, req))
}
// Another candidate (a run on request) starts its own runs.
if o := s.r.run(context.Background(), "ask-r", meshwants.StepRun{Pipeline: "run-r1.m", Module: "m", Branch: "main", Revision: 7, Run: "r1", Step: meshwants.StepRunStepComposition}); o.Outcome != meshwants.StepEndedOutcomeFailed || s.puts != 4 {
t.Errorf("a step of another candidate came to %+v with %d asks, want its own two runs", o, s.puts)
}
// A run whose ask could not be put could not run; the next step of the candidate starts it again.
f := newRunStand(t, "src2:e1", "src2:e1", "d1")
f.r.inputs.machines = func(string, string) []string { return []string{"ace"} }
put, refused := f.r.put, 0
f.r.put = func(ctx context.Context, a buildstep.StepAsk) error {
if refused == 0 {
refused++
return errors.New("no stream")
}
return put(ctx, a)
}
first := f.r.run(context.Background(), "ask-1", meshwants.StepRun{Pipeline: "main.7.m", Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
second := f.r.run(context.Background(), "ask-2", meshwants.StepRun{Pipeline: "main.7.m2", Module: "m2", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
if first.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(first.Said, "is not put") || second.Outcome != meshwants.StepEndedOutcomePassed || f.puts != 1 {
t.Errorf("after a run that could not run: %+v then %+v, %d asks", first, second, f.puts)
}
}
func mustJSON(t *testing.T, v any) []byte {
t.Helper()
raw, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
return raw
}
@@ -15,6 +15,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-assigner/internal/buckets"
@@ -441,3 +442,107 @@ func watchRuns(ctx context.Context, kv jetstream.KeyValue, r *compositionRuns, e
}()
return nil
}
// end ends the run of ask in its key with an outcome, its words and the machine's verdict, by compare-and-set; it answers
// false when the key holds another run or the run has ended already.
func (r *compositionRuns) end(key, ask, outcome, said string, verdict map[string]any) (bool, error) {
ended := false
_, _, err := r.change(key, func(c *meshwants.CompositionRun) bool {
if c.Ask != ask || c.Ended != nil {
return false
}
at := r.now().UTC()
c.Ended, c.Outcome, c.Said, c.Verdict = &at, outcome, said, verdict
ended = true
return true
})
return ended, err
}
// keyOfAsk is the key of the run whose ask is ask, or none.
func (r *compositionRuns) keyOfAsk(ask string) (string, meshwants.CompositionRun, error) {
all, err := r.kv.keys()
if err != nil {
return "", meshwants.CompositionRun{}, err
}
for _, key := range all {
if !isRunKey(key) {
continue
}
run, _, ok, err := r.read(key)
if err == nil && ok && run.Ask == ask {
return key, run, nil
}
}
return "", meshwants.CompositionRun{}, nil
}
// heardEnd ends the run of the ask a build seat's step-ended names (§719, A-240): a run ask that ended other than passed
// could not run; a passed one's answer, read with read, is the machine's verdict, or why it is none. An end of an ask no run
// names is another asker's, or of a run gone, and changes nothing.
func (r *compositionRuns) heardEnd(e buildstep.StepEnded, read func(buildstep.StepEnded) ([]byte, error)) error {
if e.Asker != runAsker || e.Ask == "" {
return nil
}
key, run, err := r.keyOfAsk(e.Ask)
if err != nil || key == "" {
return err
}
if run.Ended != nil {
return nil
}
node := run.Machine
outcome, said := meshwants.CompositionRunOutcomeCouldNotRun, ""
var verdict map[string]any
if e.Outcome != buildstep.OutcomePassed {
said = fmt.Sprintf("%s: its run ask %s ended %s on %s: %s", node, e.Ask, e.Outcome, e.Node, e.Said)
} else if answer, err := read(e); err != nil { // the program's last line of standard output (A-254)
said = node + ": " + err.Error()
} else if verdict, said = runVerdict(node, answer); said == "" {
outcome = meshwants.CompositionRunOutcomePassed
}
_, err = r.end(key, e.Ask, outcome, said, verdict)
return err
}
// await waits until each run of keys has ended, through the state's changes heard, or ctx ends: it answers each run as it
// stands then; a key gone could not run.
func (r *compositionRuns) await(ctx context.Context, keys []string) map[string]meshwants.CompositionRun {
out := map[string]meshwants.CompositionRun{}
for {
wake := r.wake()
open := false
for _, key := range keys {
run, _, ok, err := r.read(key)
switch {
case err != nil:
open = true
continue
case !ok:
at := r.now().UTC()
candidate, machine, _ := strings.Cut(key, ".")
run = meshwants.CompositionRun{Candidate: unescape(candidate), Machine: unescape(machine), Ended: &at,
Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: unescape(machine) + ": its composition run " + key + " is gone"}
case run.Ended == nil:
open = true
}
out[key] = run
}
if !open {
return out
}
select {
case <-ctx.Done():
return out
case <-wake:
}
}
}
// unescape is a key's part as a name again; a part that does not read stays as it is.
func unescape(part string) string {
if name, err := keys.UnescapeName(part); err == nil {
return name
}
return part
}
@@ -555,8 +555,9 @@ func inputsDrop(k *inputKeeper, ask func(string, any) (json.RawMessage, error))
}
// heardStepEnded deletes the input kept for the ask a build seat's step-ended names (design 47 G §354), and hands the end to the
// composition run that waits for that ask (A-240); one whose input cannot be deleted is given again by the bus.
func heardStepEnded(k *inputKeeper, ends *runEnds, m jetstream.Msg, logf func(string, ...any)) {
// composition run of that ask (A-240, §719), which writes it in the run's key; one whose input cannot be deleted, or whose run's
// end cannot be written, is given again by the bus.
func heardStepEnded(k *inputKeeper, runEnded func(buildstep.StepEnded) error, m jetstream.Msg, logf func(string, ...any)) {
var e buildstep.StepEnded
if err := json.Unmarshal(m.Data(), &e); err != nil {
logf("%s holds what is no step-ended, left: %v", m.Subject(), err)
@@ -568,8 +569,12 @@ func heardStepEnded(k *inputKeeper, ends *runEnds, m jetstream.Msg, logf func(st
_ = m.Nak()
return
}
if ends != nil {
ends.heard(e)
if runEnded != nil {
if err := runEnded(e); err != nil {
logf("the composition run of the ask %s is not ended, so the bus gives its step-ended again: %v", e.Ask, err)
_ = m.Nak()
return
}
}
_ = m.Ack()
}
@@ -11,6 +11,7 @@ import (
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"io"
"log"
@@ -527,9 +528,27 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
// The build seat's step-ended of an ask deletes the input kept for it (design 47 G §354) and ends the composition run that waits
// for it (A-240); the watchman's machine-away and machine-back say which machines a composition step does not compose (§64,
// §389). Registered before the hub starts.
ends, away := &runEnds{}, &awayNodes{}
away := &awayNodes{}
// The composition runs (design 47 G §718, §719), watched in their own state; a build seat's step-ended of a run's ask ends
// the run in its key.
var runs *compositionRuns
if kv := kvs[buckets.CompositionRuns()]; kv != nil {
set, err := runSettingsOf(getenv)
if err != nil {
return err
}
runs = newCompositionRuns(busRuns{ctx, kv}, set, time.Now, logger.Printf)
if err := watchRuns(ctx, kv, runs, nil, nil); err != nil {
return fmt.Errorf("the state %s is not watched: %w", buckets.CompositionRunsState, err)
}
}
if inputs != nil {
if err := hub.Handle(nodebuildgo.SubjectStepEndedAll, func(m jetstream.Msg) { heardStepEnded(inputs, ends, m, logger.Printf) }); err != nil {
var runEnded func(buildstep.StepEnded) error
if runs != nil && bus != nil {
read := func(e buildstep.StepEnded) ([]byte, error) { return e.ReadOutput(buildstep.AskGo, bus.CallFile) }
runEnded = func(e buildstep.StepEnded) error { return runs.heardEnd(e, read) }
}
if err := hub.Handle(nodebuildgo.SubjectStepEndedAll, func(m jetstream.Msg) { heardStepEnded(inputs, runEnded, m, logger.Printf) }); err != nil {
return err
}
}
@@ -683,17 +702,9 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
p.acts = newPutActs(bus, wake, logger.Printf)
}
// The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine.
if bus != nil && js != nil && inputs != nil && p.publishWants != nil && kvs[buckets.CompositionRuns()] != nil {
set, err := runSettingsOf(getenv)
if err != nil {
return err
}
runs := newCompositionRuns(busRuns{ctx, kvs[buckets.CompositionRuns()]}, set, time.Now, logger.Printf)
if err := watchRuns(ctx, kvs[buckets.CompositionRuns()], runs, nil, nil); err != nil {
return fmt.Errorf("the state %s is not watched: %w", buckets.CompositionRunsState, err)
}
if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil {
run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept,
trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ends: ends, ask: bus.CallFile, now: time.Now, logf: logger.Printf}
trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf}
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, now: time.Now, logf: logger.Printf,
asker: stepAskerOf(seatlist.Definition(gate.Reader, meshwants.Seat))}
if err := serveCompositionSteps(ctx, nc, steps); err != nil {