diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go index f1964a2c..d1b83bf0 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go @@ -149,6 +149,13 @@ func (r *compositionRuns) expired(boundKey string) { // endRun ends a run (end) and clears its bounds: the untaken key's readers said superseded, both bound keys deleted. func (r *compositionRuns) endRun(key, ask, outcome, said string, verdict map[string]any) (bool, error) { untaken := r.untaken(key) + // The readers a rerun carried move with the run's end (§727, A-470d): chosen before the end is written, while the rerun's own + // step still waits on it, so it alone says its own end; each mover waits for the end in the key. + if moved := r.moved.Load(); moved != nil { + if run, _, ok, err := r.read(key); err == nil && ok && run.Ended == nil && run.Ask == ask { + (*moved)(run) + } + } ended, err := r.end(key, ask, outcome, said, verdict) if err != nil || !ended { return ended, err diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go index d1873503..1643851c 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go @@ -126,7 +126,11 @@ func (r *compositionRun) run(ctx context.Context, id string, step meshwants.Step ends = append(ends, machineEnd{warnings: []string{node + " is away (the watchman's machine-away), so it is not composed"}}) continue } - j, err := r.runs.join(reader, candidate, node, buildstep.NewID()) + join := r.runs.join + if step.Rerun { // a step asked again (§727) + join = r.runs.rejoin + } + j, err := join(reader, candidate, node, buildstep.NewID()) if err != nil { ends = append(ends, machineEnd{couldNot: node + ": " + err.Error()}) continue diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go index 7024623e..8f54ac62 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go @@ -39,6 +39,8 @@ type runStand struct { trunkSeats []meshcatalog.SeatDefinitionsAnswerSeatsItem unjudged string listFault error + // verdict, when set, is the composer's verdict of a machine (by node), else the stand's own. + verdict func(node string) (composer.Machine, bool) } func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *runStand { @@ -127,7 +129,13 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) * s.asks[node], s.requests[node] = a, req s.mu.Unlock() m := composer.Machine{Composes: true} - if node == s.unjudged { + override, set := composer.Machine{}, false + if s.verdict != nil { + override, set = s.verdict(node) + } + if set { + m = override + } else if node == s.unjudged { m = composer.Machine{NotJudged: "the composer was handed no seat list"} } else if node == "g14" { m = composer.Machine{Reason: "two modules own one path", Problems: []string{"/etc/x is m's and bad's"}} @@ -452,3 +460,92 @@ func TestAMachineIsJudgedByTheModulesItBlames(t *testing.T) { }) } } + +// Design 47 G §727, A-468g, A-470d: a rerun of a composition step starts a new run only on the machine it did not pass, replacing +// that key and carrying every former reader, the one that did not pass marked rerun with the rerun's caller; at the new run's end +// that reader says its end again, marked rerun, naming the new run's ask; a reader that passed does not move; the machine the +// step passed keeps its verdict. +func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) { + s := newRunStand(t, "src2:e1", "src2:e1", "d1") + s.r.inputs.machines = func(string, string) []string { return []string{"ace", "g14"} } + var mu sync.Mutex + fixed := false + s.verdict = func(node string) (composer.Machine, bool) { + mu.Lock() + defer mu.Unlock() + m := composer.Machine{Composes: true} + m.Blamed = []struct { + Module string `json:"module"` + Problem string `json:"problem"` + }{} + if node == "g14" && !fixed { + m.Composes, m.Reason = false, "two modules own one path" + m.Blamed = append(m.Blamed, struct { + Module string `json:"module"` + Problem string `json:"problem"` + }{"a", "/etc/x"}, struct { + Module string `json:"module"` + Problem string `json:"problem"` + }{"d", "/etc/x"}) + } + return m, true + } + ends := make(chan meshwants.StepEnded, 16) + steps := &compositionSteps{node: "n1", now: time.Now, logf: t.Logf, run: s.r.run, resume: s.r.resume, + say: func(event string, body any, _ string) error { + if e, ok := body.(meshwants.StepEnded); ok { + ends <- e + } + return nil + }} + move := steps.move(context.Background()) + s.runs.moved.Store(&move) + step := func(m string, rerun bool) []byte { + raw, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.7." + m, Module: m, Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition, Rerun: rerun, By: map[bool]string{true: "op@n1"}[rerun]}) + return raw + } + got := map[string]meshwants.StepEnded{} + for _, m := range []string{"a", "b", "d"} { + steps.take(context.Background(), "ask-"+m, step(m, false)) + e := <-ends + got[e.Module] = e + } + if got["a"].Outcome != meshwants.StepEndedOutcomeFailed || got["d"].Outcome != meshwants.StepEndedOutcomeFailed || got["b"].Outcome != meshwants.StepEndedOutcomePassed { + t.Fatalf("the first ends: %+v", got) + } + former := got["d"].Shared[1].Ask + mu.Lock() + fixed = true + mu.Unlock() + steps.take(context.Background(), "ask-a2", step("a", true)) + moved := map[string]meshwants.StepEnded{} + for range 2 { + select { + case e := <-ends: + moved[e.ID] = e + case <-time.After(5 * time.Second): + t.Fatalf("the ends after the rerun: %+v", moved) + } + } + a, d := moved["ask-a2"], moved["ask-d"] + if a.Outcome != meshwants.StepEndedOutcomePassed || d.Outcome != meshwants.StepEndedOutcomePassed || !d.Rerun || d.By != "op@n1" || + len(d.Shared) != 2 || d.Shared[1].Machine != "g14" || d.Shared[1].Ask == former || d.Shared[1].Ask != a.Shared[1].Ask { + t.Fatalf("after the rerun: a %+v, d %+v", a, d) + } + if s.puts != 3 { + t.Errorf("%d run asks, want one more for the machine a did not pass", s.puts) + } + select { + case e := <-ends: + t.Errorf("a reader that passed moved: %+v", e) + case <-time.After(50 * time.Millisecond): + } + run, _, _, _ := s.runs.read(runKeyOf("main.7", "g14")) + marks := map[string]bool{} + for _, c := range run.Readers { + marks[c.ID] = c.Rerun + } + if len(run.Readers) != 3 || !marks["ask-d"] || marks["ask-b"] || !marks["ask-a2"] { + t.Errorf("the new run's readers %+v", run.Readers) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go index e3a223cc..58015a2e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go @@ -10,6 +10,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "time" "github.com/nats-io/nats.go/jetstream" @@ -222,6 +223,9 @@ type compositionRuns struct { // nil says or drops nothing. say publisher drop func(ask, why string) error + // moved is told each run that ended, for the readers a rerun carried to say their end again (compositionSteps.move). + // Set once the steps are served, after the hub starts. + moved atomic.Pointer[func(run meshwants.CompositionRun)] } func newCompositionRuns(kv runsKV, set runSettings, now func() time.Time, logf func(string, ...any)) *compositionRuns { @@ -579,9 +583,9 @@ func (r *compositionRuns) sharedOf(keys []string, runs map[string]meshwants.Comp } // leave removes the composition steps of a superseded or cancelled pipeline from every run's readers (§723, A-469n), and answers -// their asks' ids; a running run left with no reader has its ask dropped and ends could not run, "no reader is left" (A-469z), +// their asks' ids, each told to stop before a run is ended; a running run left with no reader has its ask dropped and ends could not run, "no reader is left" (A-469z), // so a later step of the candidate starts a new run. A run other readers read goes on for them. -func (r *compositionRuns) leave(pipeline string) ([]string, error) { +func (r *compositionRuns) leave(pipeline string, stop func(ids []string)) ([]string, error) { all, err := r.kv.keys() if err != nil { return nil, err @@ -614,6 +618,9 @@ func (r *compositionRuns) leave(pipeline string) ([]string, error) { continue } ids = append(ids, left...) + if stop != nil { + stop(left) // the steps that left say no end, the run's included + } if run.Ended != nil || len(run.Readers) > 0 { continue } @@ -629,3 +636,96 @@ func (r *compositionRuns) leave(pipeline string) ([]string, error) { } return ids, errors.Join(errs...) } + +// passedOn is whether the step of module passed on the machine of an ended run (§722): neither failed nor could not run there. +func passedOn(module string, run meshwants.CompositionRun) bool { + e := judgeRun(module, run) + return e.failed == "" && e.couldNot == "" +} + +// formerOf is the place in readers of a step's former ask: the reader of the same pipeline's composition step, or -1. +func formerOf(readers []meshwants.CompositionReader, pipeline string) int { + return slices.IndexFunc(readers, func(c meshwants.CompositionReader) bool { + return c.Pipeline == pipeline && c.Step == meshwants.CompositionReaderStepComposition + }) +} + +// rejoin is join for a step asked again (§727, A-468g, A-470d): on a machine whose ended run it passed it keeps that verdict, +// reading the run under its new ask; a running run it joins; an ended run it did not read and that could run it reads, as one +// started after its last end; otherwise it starts a new run, replacing the ended key by compare-and-set and carrying every former +// reader into the new value: each that did not pass on the former run marked `rerun` with by the rerun's caller, each that passed +// not (A-470d). +func (r *compositionRuns) rejoin(reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) { + key := runKeyOf(candidate, machine) + for { + run, rev, ok, err := r.read(key) + if err != nil { + return joined{}, err + } + if !ok { + return r.join(reader, candidate, machine, ask) + } + at := formerOf(run.Readers, reader.Pipeline) + ended := run.Ended != nil + keep := ended && at >= 0 && passedOn(reader.Module, run) + read := !ended || (at < 0 && run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun) + if keep || read { + if at >= 0 && run.Readers[at].ID == reader.ID { + return joined{key: key, run: run}, nil + } + if at >= 0 { + run.Readers[at] = reader + } else { + run.Readers = append(run.Readers, reader) + } + if err := r.write(key, run, rev); errors.Is(err, errRunChanged) { + continue + } else if err != nil { + return joined{}, err + } + return joined{key: key, run: run}, nil + } + if err := r.room(); err != nil { + return joined{}, err + } + started := r.now().UTC() + fresh := meshwants.CompositionRun{Candidate: candidate, Machine: machine, Ask: ask, Started: &started} + for i, c := range run.Readers { + if i == at { + continue + } + c.Rerun, c.By = !passedOn(c.Module, run), "" + if c.Rerun { + c.By = reader.By + } + fresh.Readers = append(fresh.Readers, c) + } + fresh.Readers = append(fresh.Readers, reader) + if err := r.write(key, fresh, rev); errors.Is(err, errRunChanged) { + continue + } else if err != nil { + return joined{}, err + } + for _, k := range []string{runBoundPrefix + key, runUntakenPrefix + key} { + _ = r.kv.del(k) // the former run's bounds go with it + } + if err := r.kv.arm(runUntakenPrefix+key, r.set.untaken); err != nil { + r.logf("the untaken bound of the composition run %s is not armed: %v", key, err) + } + r.sayWaits(fresh, fresh.Readers, "") + return joined{key: key, started: true, run: fresh}, nil + } +} + +// write writes a run's value at its revision. +func (r *compositionRuns) write(key string, run meshwants.CompositionRun, rev uint64) error { + raw, err := json.Marshal(run) + if err != nil { + return err + } + if _, err := r.kv.update(key, raw, rev); err != nil { + return err + } + r.heard() + return nil +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go index fa6e9208..5c1089aa 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go @@ -136,6 +136,11 @@ func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.St if c.stops == nil { c.stops = map[string]context.CancelFunc{} } + if c.stops[id] != nil { // the step asked under id is running already: it says its own end + c.mu.Unlock() + stopWait() + return + } c.stops[id] = stopWait c.mu.Unlock() defer func() { @@ -161,6 +166,27 @@ func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.St } } +// move says the end again of each reader a rerun carried into an ended run, marked rerun (§727, A-470d): its machines judged +// from the candidate's keys naming it, the run's new ask in `shared`. A reader running here says its own end. +func (c *compositionSteps) move(ctx context.Context) func(run meshwants.CompositionRun) { + return func(run meshwants.CompositionRun) { + for _, reader := range run.Readers { + if !reader.Rerun || c.running(reader.ID) { + continue + } + c.logf("the composition step %s of %s moves with the composition run of %s (ask %s)", reader.ID, reader.Pipeline, run.Machine, run.Ask) + go c.say1(ctx, reader.ID, stepRunOf(reader), c.resume) + } + } +} + +// running is whether the step asked under id runs here. +func (c *compositionSteps) running(id string) bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.stops[id] != nil +} + // leave ends the waits of the steps asked under ids (§723: a superseded or cancelled step leaves its runs). func (c *compositionSteps) leave(ids []string) { c.mu.Lock() diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go index 7057b980..07e25cb8 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go @@ -171,19 +171,18 @@ func TestASupersededStepLeavesItsRuns(t *testing.T) { } key := runKeyOf("main.3", "n1") for { + wake := r.wake() if run, _, _, _ := r.read(key); len(run.Readers) == 2 { break } - <-r.wake() + <-wake } - ids, err := r.leave("main.3.a") - c.leave(ids) + ids, err := r.leave("main.3.a", c.leave) run, _, _, _ := r.read(key) if err != nil || !slices.Equal(ids, []string{"ask-main.3.a"}) || run.Ended != nil || len(run.Readers) != 1 || len(drops) != 0 { t.Fatalf("after a's supersede: %v %v, run %+v, drops %v: want the run going on for b", ids, err, run, drops) } - ids, err = r.leave("main.3.b") - c.leave(ids) + _, err = r.leave("main.3.b", c.leave) run, _, _, _ = r.read(key) if err != nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || !strings.Contains(run.Said, "no reader is left") || len(drops) != 1 || !strings.HasPrefix(drops[0], run.Ask+": no reader is left") { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 72d2da70..6149ec14 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -714,6 +714,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf} steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume, now: time.Now, logf: logger.Printf, asker: stepAskerOf(seatlist.Definition(gate.Reader, meshwants.Seat))} + move := steps.move(ctx) // the readers a rerun carried (§727, A-470d) + runs.moved.Store(&move) // The state's watch wakes the steps waiting on a run and hears a bound's expiry; the runs standing at the start are taken up // once the values that stand are read (§719, §720). if err := watchRuns(ctx, kvs[buckets.CompositionRuns()], runs, runs.expired, func() { steps.takeUp(ctx, runs) }); err != nil { @@ -722,8 +724,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, // A superseded or cancelled pipeline's composition step leaves the runs it reads (§723, A-469n). if p.ends != nil { left := func(pipeline string) error { - ids, err := runs.leave(pipeline) - steps.leave(ids) + _, err := runs.leave(pipeline, steps.leave) return err } p.ends.left.Store(&left)