From 0f167db73dff05fee3e771823e44c57bcca5fe85 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 11:10:01 +0100 Subject: [PATCH 1/8] Rerun of validate or composition asks the shared ask (H-A-236) The rerun verb looked a step's ask up by the mover's id, but the validate ask is one per pipeline and names no mover, so every validate rerun, and a composition rerun, was refused "placed no ask now". Builds: design 47 H H-A-236 --- .../internal/pipeline/verbs.go | 16 +++++++ .../internal/pipeline/verbs_test.go | 42 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index d86cec3b..6ccd0f0b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -509,6 +509,22 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time if !ok { return nil, fmt.Errorf("%s moves no module %q", p.Pipeline, s.Module) } + // The validate ask is the pipeline's one, naming no mover (H-A-51): its rerun asks that one ask again; a composition's + // rerun builds the mover's composition ask directly, never looked up among the placed steps (H-A-236). + switch s.Step { + case seat.StepResultStepValidate, seat.StepResultStepComposition: + if s.Seat == wants.Seat { + id, step := AskID(now, p.Pipeline, "", "", seat.StepResultStepValidate), wants.StepRunStepValidate + if s.Step == seat.StepResultStepComposition { + id, step = AskID(now, p.Pipeline, m.Module, "", seat.StepResultStepComposition), wants.StepRunStepComposition + } + a := wantsAsk(p, m, id, step) + b := a.Ask.(wants.StepRun) + b.Rerun, b.By = true, caller + a.Ask = b + return &a, nil + } + } reg, ok := v.Starter.Registration(m.Module) if !ok { return nil, fmt.Errorf("%s has no registration", m.Module) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go index a2c8adae..ed9a67c3 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go @@ -8,6 +8,7 @@ import ( catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" "git.novox.be/novox/mesh-sdk/go/views" ) @@ -252,6 +253,47 @@ func TestRerunOfARunAsksWithItsCandidatesHalf(t *testing.T) { } } +// H-A-236: a rerun of a failed pipeline's validate step asks the pipeline's one validate ask again (H-A-51), naming no +// mover in its id; a rerun of a composition step builds the mover's composition ask directly. Neither is refused "placed no +// ask now". +func TestRerunOfValidateOrCompositionAsksTheSharedAsk(t *testing.T) { + v, store, _, asks, _, _ := verbsFixture() + reg, entries := placeFixture() + reg.Policy.Steps.Validate, reg.Policy.Steps.Composition = "required", "required" + v.Starter.Registration = func(string) (catalog.Registration, bool) { return reg, true } + p, rev, _ := store.Get(pkey) + p.Steps, _ = Place(p, p.Movers[0], reg, entries, time.Unix(1, 0), "the-runner") + for i := range p.Steps { + switch p.Steps[i].Step { + case seat.StepResultStepValidate: + p.Steps[i].Outcome = seat.StepResultOutcomeFailed + case seat.StepResultStepComposition: + p.Steps[i].Outcome = seat.StepResultOutcomeNotAsked + } + } + p.Outcome = seat.PipelineOutcomeFailed + store.Update(pkey, p, rev) + now := v.now() + for _, c := range []struct{ step, module string }{{seat.StepResultStepValidate, ""}, {seat.StepResultStepComposition, "app"}} { + n := len(asks.asks) + r, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Module: "app", Step: c.step, Why: "the holder restarted"}) + want := AskID(now, pkey, c.module, "", c.step) + if err != nil || r.Result == nil || r.Result.Ask != want || r.Result.Outcome != seat.StepResultOutcomeRunning { + t.Fatalf("a rerun of %s = %+v, %v, want running under %s", c.step, r.Result, err, want) + } + if len(asks.asks) != n+1 || asks.ids[n] != want { + t.Fatalf("a rerun of %s asked %v, want one ask under %s", c.step, asks.ids[n:], want) + } + b, ok := asks.asks[n].(wants.StepRun) + if !ok || b.Step != c.step || b.Module != "app" || b.Pipeline != pkey || !b.Rerun || b.By != "operator" || b.Asker != seat.Seat { + t.Errorf("the %s ask = %+v", c.step, asks.asks[n]) + } + p, rev, _ = store.Get(pkey) + p.Steps[stepIndex(p, c.step, "")].Outcome = seat.StepResultOutcomeFailed + store.Update(pkey, p, rev) + } +} + func stepIndex(p seat.Pipeline, step, artifact string) int { for i, s := range p.Steps { if s.Step == step && s.Artifact == artifact { From b95b3ef989058c531308401123e9c7eba78aaf59 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 11:23:31 +0100 Subject: [PATCH 2/8] Bound every ask the runner puts on a seat (H-A-235) An ask whose holder lost it (the seat mesh-wants' holder restarted) held its pipeline's turn for nine hours. Every ask but a test run's now arms a key of the state bounds before it is put; its expiry asks once more, said in the journal, and the second ends the step could not run with the finding call-unanswered, so the pipeline ends and its turn frees. Builds: design 47 H H-A-235, H-A-238 (1)-(5) --- .../cmd/mesh-delivery-pipeline/askbounds.go | 114 +++++++++ .../cmd/mesh-delivery-pipeline/drops.go | 4 +- .../cmd/mesh-delivery-pipeline/main.go | 34 ++- .../queue_integration_test.go | 27 ++- .../cmd/mesh-delivery-pipeline/steps.go | 11 +- .../testruns_integration_test.go | 8 +- .../internal/pipeline/asks.go | 228 ++++++++++++++++++ .../internal/pipeline/asks_test.go | 102 ++++++++ .../internal/pipeline/bounds.go | 4 +- .../internal/pipeline/place.go | 16 +- .../internal/pipeline/start.go | 4 +- .../internal/pipeline/steps.go | 4 + .../internal/pipeline/steps_test.go | 2 +- .../internal/pipeline/supersede.go | 3 + .../internal/pipeline/verbs.go | 61 +++-- .../mesh-delivery-pipeline/manifest_test.go | 4 +- modules/mesh-delivery-pipeline/module.json | 16 ++ 17 files changed, 583 insertions(+), 59 deletions(-) create mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go create mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/asks.go create mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go new file mode 100644 index 00000000..b91e9069 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go @@ -0,0 +1,114 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/nats-io/nats.go/jetstream" + + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// envAskUnansweredBound names the file the manifest writes the setting ask-unanswered-bound into, `${config:}` +// (design 47 H H-A-238 (1)). +const envAskUnansweredBound = "MESH_RUNNER_ASK_UNANSWERED_BOUND_FILE" + +// askUnansweredBound reads the setting ask-unanswered-bound at the start, a Go duration (H-A-235, H-A-238 (1)): its default +// ten minutes when the file is missing or empty, an error naming the file for a content that is no positive duration. +func askUnansweredBound(getenv func(string) string) (time.Duration, error) { + v, err := settingText(getenv(envAskUnansweredBound)) + if err != nil || v == "" { + return pipeline.DefaultAskUnansweredBound, err + } + d, err := time.ParseDuration(v) + if err != nil || d <= 0 { + return 0, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(envAskUnansweredBound), v) + } + return d, nil +} + +// askClock is the bound of every ask but a test run's over the state bounds (H-A-238 (5)): a key armed with its own time to +// live before the ask is put, deleted at its holder's first word, its expiry heard by watchBounds; the finding call-unanswered, +// scope call, id the ask, raised at the second bound and cleared when the ask is answered after all (H-A-238 (2)). +func askClock(ctx context.Context, bounds jetstream.KeyValue, bound time.Duration, w callFindings, module string, now func() time.Time) *pipeline.AskClock { + return &pipeline.AskClock{Bound: bound, + Arm: func(key string, ttl time.Duration) error { + _, err := bounds.Create(ctx, key, []byte(`{"ask":"`+key+`"}`), jetstream.KeyTTL(ttl)) + if errors.Is(err, jetstream.ErrKeyExists) { + return nil // armed already: the same ask put again at a start keeps its first bound + } + return err + }, + // A first word is heard for every ask, most of them unbounded or answered already: a key not there is deleted of + // nothing and writes no marker. + Unbound: func(key string) error { + if _, err := bounds.Get(ctx, key); errors.Is(err, jetstream.ErrKeyNotFound) { + return nil + } else if err != nil { + return err + } + if err := bounds.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err + } + return nil + }, + Raise: func(onSeat, ask string, within time.Duration) error { + if w == nil { + return errors.New("the runner has no finding writer") + } + return w.Write(ctx, condseat.Finding{Scope: mcc.ScopeCall, ID: ask, Kind: mcc.KindCallUnanswered, Severity: condseat.FindingSeverityWarning, + Summary: fmt.Sprintf("the ask %s of the seat %s is unanswered", ask, onSeat), + Headline: "A step's ask is not answered", + Explanation: fmt.Sprintf("The runner put the ask %s on the seat %s, asked it once more, and heard no word of its holder within %s, so its step ended could not run and its pipeline went on (H-A-235).", ask, onSeat, within), + Source: module + "." + mcc.KindCallUnanswered, Observer: module, At: now().UTC(), Subject: condseat.Subject{Scope: mcc.ScopeCall, ID: ask}}) + }, + Clear: func(_, ask string) error { + if w == nil { + return nil + } + return w.Clear(ctx, mcc.ScopeCall, ask, mcc.KindCallUnanswered, "the ask is answered") + }} +} + +// answered deletes the bounds of the ask on onSeat at its holder's first word, said when it fails: the bound then passes and +// its expiry finds the step answered or asks it once more. +func answered(c *pipeline.AskClock, onSeat, ask string, logf func(string, ...any)) { + if err := c.Answered(onSeat, ask); err != nil { + logf("the bounds of ask %s on %s are not deleted at its holder's word: %v", ask, onSeat, err) + } +} + +// hearWantsProgress takes a step progress of the seat mesh-wants as its holder's first word on the ask it names (H-A-238 (3)). +func hearWantsProgress(hub handlers, c *pipeline.AskClock, logf func(string, ...any)) error { + return hub.Handle(wants.SubjectStepProgressAll, func(m jetstream.Msg) { + var ev wants.StepProgressEvent + if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" { + logf("%s is no step-progress: %v", m.Subject(), err) + _ = m.Term() + return + } + answered(c, wants.Seat, ev.Ask, logf) + _ = m.Ack() + }) +} + +// askBoundExpired takes the expiry of an ask's bound (H-A-235): the first asks once more, the second ends its step. +func askBoundExpired(unanswered func(onSeat, ask string, again bool) error, logf func(string, ...any)) func(key string) bool { + return func(key string) bool { + s, ask, again, ok := pipeline.ParseAskBound(key) + if !ok { + return false + } + if err := unanswered(s, ask, again); err != nil { + logf("the bound of ask %s on %s passed and is not taken: %v", ask, s, err) + } + return true + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops.go index 1915061c..b3a3c76f 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops.go @@ -72,8 +72,8 @@ func dropUnanswered(ctx context.Context, w callFindings, module string, now func } } -// boundExpired takes the expiry of a key of the state bounds: a drop's says the drop unanswered (A-222); another key is none -// of the runner's, an ask having no bound of its own (§535, §647 as A-466y amends them). +// boundExpired takes the expiry of a drop's key of the state bounds: it says the drop unanswered (A-222); an ask's key is +// taken by askBoundExpired (H-A-235). func boundExpired(unanswered func(buildSeat, ask string) error, logf func(string, ...any)) func(key string) { return func(key string) { if s, ask, ok := pipeline.ParseDropBound(key); ok { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 1597d3e7..378dc72d 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -67,8 +67,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - // The bound of each superseded step's drop: a key with its own time to live, its expiry heard (A-222, A-77). - bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound a drop no holder answers") + // The bound of each superseded step's drop and of each ask: a key with its own time to live, its expiry heard (A-222, A-77; + // H-A-238 (5)). + bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound a drop or an ask no holder answers") if err != nil { return err } @@ -160,6 +161,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains), Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing} hearer.AskSteps = starter.AskSteps + // Every ask but a test run's is bounded until its holder's first word (H-A-235, H-A-238), its bounds keys of the state + // bounds. + askBound, err := askUnansweredBound(getenv) + if err != nil { + return err + } + logger.Printf("asks: each unanswered ask bound %s, asked once more, then its step ends could not run", askBound) + clock := askClock(ctx, bounds, askBound, rv.Findings, cred.Module, time.Now) + starter.Asks, hearer.Asks = clock, clock starter.Supersede = func(newer seat.Pipeline) { hearer.SupersedeRunning(newer) } hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) } hearer.AskPinHeld = func(key, module string, pins map[string]string) { @@ -297,10 +307,23 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } return hearer.RunTaken(ask, node) } - if err := hearAskTaken(heard, taken, logger.Printf); err != nil { + if err := hearAskTaken(heard, taken, clock, logger.Printf); err != nil { return err } - expired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf) + if err := hearWantsProgress(heard, clock, logger.Printf); err != nil { + return err + } + // The seat's write verbs are answered by the process that holds the pipelines (A-107); made before the bounds are heard, + // since an ask's expired bound asks it once more through them (H-A-235). + verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls) + verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill} + dropExpired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf) + askExpired := askBoundExpired(verbs.AskUnanswered, logger.Printf) + expired := func(key string) { + if !askExpired(key) { + dropExpired(key) + } + } if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil { return err } @@ -336,9 +359,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The steps' ends are kept seat events, each on its own stream (step-ended of the build seats, mesh-builds and // mesh-wants), never on EVENTS: bound here, or no step of any pipeline is ever heard ending (G-1 §24). logger.Printf("the kept seat events heard: %d streams bound", hub.StartKept(ctx, js, cred)) - // The seat's write verbs are answered here, by the process that holds the pipelines (A-107). - verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls) - verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill} stopVerbs, err := answerVerbs(calls, seats, verbs) if err != nil { return err diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go index 8e9fb2d3..5ad1481c 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go @@ -153,6 +153,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { if err := os.WriteFile(waitingCap, []byte("1\n"), 0o600); err != nil { t.Fatal(err) } + askBound := filepath.Join(dir, "ask-unanswered-bound") logs := &syncBuffer{} start := func() func() { runCtx, stop := context.WithCancel(ctx) @@ -164,6 +165,8 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { return broker case envPipelinesWaitingCap: return waitingCap + case envAskUnansweredBound: + return askBound } return "" }, logs) @@ -359,9 +362,12 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { } // At the runner's next start the waiting main.10 is placed from its own state, saying wait-ended, which clears - // pipelines-stalled; the start begins no pipeline of its own. + // pipelines-stalled; the start begins no pipeline of its own. That start's ask-unanswered-bound is one second. stopRunner() setMaxMsgSize(t, ctx, js, "KV_"+module+"_"+seat.StateQueue, -1) + if err := os.WriteFile(askBound, []byte("1s\n"), 0o600); err != nil { + t.Fatal(err) + } before := pipelineKeys(t, ctx, pipelines) stopRunner = start() outcome("main.10", seat.PipelineOutcomeRunning) @@ -370,6 +376,25 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { if after := pipelineKeys(t, ctx, pipelines); !slices.Equal(before, after) { t.Fatalf("the start wrote pipelines %v, want %v", after, before) } + + // H-A-235, H-A-238: main.10's build ask, which no holder answers, is asked once more when its bound's key expires, and + // at its second bound's expiry its step ends could not run, the pipeline ends failed and frees its turn, and the finding + // call-unanswered names the ask. + build := stepFor(get("main.10"), "app", seat.StepResultStepBuild) + if build.Ask == "" { + t.Fatalf("main.10's build = %+v, want it asked", build) + } + waitFor(t, ctx, logs, func() bool { + return strings.Contains(logs.String(), "the ask "+build.Ask+" of the build step of main.10 on mesh-builds had no word from its holder within 1s: asked once more") + }) + outcome("main.10", seat.PipelineOutcomeFailed) + if s := stepFor(get("main.10"), "app", seat.StepResultStepBuild); s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build.Ask+" was not answered" { + t.Fatalf("main.10's build = %+v, want could not run, its ask not answered", s) + } + waitFor(t, ctx, logs, func() bool { + e, err := findings.Get(ctx, "call."+build.Ask+".call-unanswered") + return err == nil && strings.Contains(string(e.Value()), build.Ask) && len(running()) == 0 + }) } func stepFor(p seat.Pipeline, module, step string) seat.StepResult { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go index c4aa18c1..81b53a6a 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -141,6 +141,8 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any)) logf("the step-ended of ask %s is not taken yet, heard again: %v", ev.ID, err) _ = m.Nak() default: + // The step's end is its holder's word on the ask, the first for mesh-builds (H-A-238 (3)). + answered(h.Asks, bs.Seat, ev.ID, logf) _ = m.Ack() } }) @@ -153,8 +155,9 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any)) // hearAskTaken keeps on its step the node whose build seat holder took an ask (§602, §647; A-182), so a drop of it is asked of // that node's holder, and begins the clock of a test run's ask (§725). -func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(string, ...any)) error { +func hearAskTaken(hub handlers, taken func(ask, node string) error, c *pipeline.AskClock, logf func(string, ...any)) error { for _, ts := range pipeline.TakenSeats { + ts := ts err := hub.Handle(ts.Subject, func(m jetstream.Msg) { var ev nodebuildgo.AskTaken if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" { @@ -167,6 +170,8 @@ func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(st _ = m.Nak() return } + // The seat's ask-taken is its holder's first word on the ask (H-A-238 (3)). + answered(c, ts.Seat, ev.Ask, logf) _ = m.Ack() }) if err != nil { @@ -176,8 +181,8 @@ func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(st return nil } -// watchBounds hears the end of each drop's bound, the bus's marker of the key's expiry, and says a drop no holder answered -// unanswered (A-222, A-77): no timer in the process. +// watchBounds hears the end of each drop's and each ask's bound, the bus's marker of the key's expiry (A-222, A-77; H-A-235, +// H-A-238 (5)): no timer in the process. func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error { w, err := bounds.WatchAll(ctx, jetstream.UpdatesOnly()) if err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go index e39fabf9..182cb70f 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go @@ -350,7 +350,8 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { if s := step(read(p3), "app:tools", "unit"); s.Outcome != seat.StepResultOutcomeWaitingForABuildNode || s.Ask != "" || s.Shared[0].Ask != unit.Ask { t.Fatalf("the tools' unit step = %+v; want it waiting for a build node, reading the program's run", s) } - // Its wait has no bound (§647 as A-466y amends it): the state bounds holds no key for a queued ask, only drops' keys. + // A test run's ask keeps §725's bounds alone (H-A-238 (4)): the state bounds holds no key naming it, only drops' and other + // asks' keys. boundsKV, err := js.KeyValue(ctx, module+"_bounds") if err != nil { t.Fatal(err) @@ -359,8 +360,9 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { t.Fatal(err) } else { for _, k := range ks { - if !strings.HasPrefix(k, pipeline.DropBoundPrefix) { - t.Errorf("the state bounds holds %s after the asks were put; want no bound of a queued ask", k) + _, ask, _, isAsk := pipeline.ParseAskBound(k) + if (!isAsk && !strings.HasPrefix(k, pipeline.DropBoundPrefix)) || ask == unit.Ask || ask == integ.Ask { + t.Errorf("the state bounds holds %s after the asks were put; want no bound of a test run's ask", k) } } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/asks.go b/modules/mesh-delivery-pipeline/internal/pipeline/asks.go new file mode 100644 index 00000000..3f009af3 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/asks.go @@ -0,0 +1,228 @@ +package pipeline + +import ( + "errors" + "fmt" + "strings" + "time" + + builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" +) + +// DefaultAskUnansweredBound is the setting ask-unanswered-bound's default (design 47 H H-A-235, H-A-238 (1)): ten minutes +// from an ask until its holder's first word. +const DefaultAskUnansweredBound = 10 * time.Minute + +// The keys of the state bounds that time an ask the runner put (H-A-238 (5)): `ask..` armed before the ask is put, +// `ask2..` armed when it is asked once more, each with the setting as its own time to live. +const ( + AskBoundPrefix = "ask." + AskAgainBoundPrefix = "ask2." +) + +// AskBoundKey is the key of the first bound of the ask on onSeat; AskAgainBoundKey that of its second. +func AskBoundKey(onSeat, ask string) string { return AskBoundPrefix + onSeat + "." + ask } +func AskAgainBoundKey(onSeat, ask string) string { return AskAgainBoundPrefix + onSeat + "." + ask } + +// ParseAskBound reads a key of the state bounds as an ask's bound: the seat, the ask and whether it is the second bound. +func ParseAskBound(key string) (onSeat, ask string, again, ok bool) { + rest, found := strings.CutPrefix(key, AskAgainBoundPrefix) + again = found + if !found { + if rest, found = strings.CutPrefix(key, AskBoundPrefix); !found { + return "", "", false, false + } + } + onSeat, ask, found = strings.Cut(rest, ".") + return onSeat, ask, again, found && onSeat != "" && ask != "" && !strings.Contains(ask, ".") +} + +// UnansweredWhy is the why of a step whose ask was not answered within its two bounds (H-A-235). +func UnansweredWhy(ask string) string { return "its ask " + ask + " was not answered" } + +// AskClock bounds every ask the runner puts on a seat but a test run's, which keeps §725's bounds (H-A-235, H-A-238 (4)): +// Arm writes a key of the state bounds with Bound as its own time to live, Unbound deletes one; Raise raises the finding +// call-unanswered of an ask that ended its step, Clear clears it once the ask is answered after all (H-A-238 (2)). +type AskClock struct { + Bound time.Duration + Arm func(key string, ttl time.Duration) error + Unbound func(key string) error + Raise func(onSeat, ask string, bound time.Duration) error + Clear func(onSeat, ask string) error +} + +// arm arms the first bound of a, before it is put (H-A-238 (5)); a test run's ask, and a nil clock, arm none. +func (c *AskClock) arm(a Asked) error { + if c == nil || c.Arm == nil || a.Run != "" { + return nil + } + if err := c.Arm(AskBoundKey(a.Seat, a.ID), c.bound()); err != nil { + return fmt.Errorf("the bound of ask %s is not armed: %w", a.ID, err) + } + return nil +} + +func (c *AskClock) bound() time.Duration { + if c.Bound > 0 { + return c.Bound + } + return DefaultAskUnansweredBound +} + +// Answered deletes both bounds of the ask on onSeat, its holder's first word heard: an ask-taken, a step progress or the +// step's end (H-A-238 (3)). +func (c *AskClock) Answered(onSeat, ask string) error { + if c == nil || c.Unbound == nil || ask == "" { + return nil + } + return errors.Join(c.Unbound(AskBoundKey(onSeat, ask)), c.Unbound(AskAgainBoundKey(onSeat, ask))) +} + +// answeredLate clears the finding of the ask of step s, which ended unanswered, its holder heard after all (H-A-238 (2)). +func (c *AskClock) answeredLate(s seat.StepResult, logf func(string, ...any)) { + if c == nil || c.Clear == nil || s.Ask == "" || s.Why != UnansweredWhy(s.Ask) { + return + } + if err := c.Clear(s.Seat, s.Ask); err != nil { + logf("the finding of ask %s, answered after its step ended, is not cleared: %v", s.Ask, err) + } +} + +// AskUnanswered takes the expiry of a bound of the ask on onSeat (H-A-235): at the first the ask is put once more under its +// own id, said in the journal, its second bound armed; at the second the step ends could not run "its ask was not +// answered", the finding call-unanswered raised, so the pipeline ends and its turn frees. A bound whose ask no running step +// names any longer asks nothing. +func (v Verbs) AskUnanswered(onSeat, ask string, again bool) error { + h, c := v.Hearer, v.Starter.Asks + if c == nil { + return nil + } + key, p, i, err := v.runningAsk(onSeat, ask) + if err != nil || key == "" { + return err + } + s := p.Steps[i] + if again { + h.logf("the ask %s of the %s step of %s on %s had no word from its holder within its two bounds: the step ends could not run (H-A-235)", + ask, s.Step, key, onSeat) + if err := h.endUnanswered(key, ask); err != nil { + return err + } + if c.Raise != nil { + return c.Raise(onSeat, ask, 2*c.bound()) + } + return nil + } + a, err := v.askAgain(p, s, h.now()) + if err != nil { + h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s and cannot be put again: %v", ask, s.Step, key, onSeat, c.bound(), err) + } else { + h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s: asked once more (H-A-235)", ask, s.Step, key, onSeat, c.bound()) + } + if c.Arm != nil { + if aerr := c.Arm(AskAgainBoundKey(onSeat, ask), c.bound()); aerr != nil { + return fmt.Errorf("the second bound of ask %s is not armed, so it is not asked again: %w", ask, aerr) + } + } + if err != nil { + return nil // its second bound ends the step + } + v.Starter.carry(key, []Asked{*a}) + return askAll(v.Starter.Steps, key, []Asked{*a}, func(commit string) bool { return v.Starter.recovered(key, commit) }, nil) +} + +// runningAsk is the pipeline and the index of its running step asked under ask of onSeat; an empty key when none is. +func (v Verbs) runningAsk(onSeat, ask string) (string, seat.Pipeline, int, error) { + pkeys, err := allKeys(v.Hearer.Store) + if err != nil { + return "", seat.Pipeline{}, -1, fmt.Errorf("the pipelines are not listed: %w", err) + } + for _, key := range pkeys { + p, _, err := v.Hearer.Store.Get(key) + if err != nil { + continue + } + if i := stepOfAsk(p, ask); i >= 0 && open(p.Steps[i]) && p.Steps[i].Seat == onSeat { + return key, p, i, nil + } + } + return "", seat.Pipeline{}, -1, nil +} + +// askAgain is the ask of running step s as it was put: its own id, a rerun's marks kept. +func (v Verbs) askAgain(p seat.Pipeline, s seat.StepResult, now time.Time) (*Asked, error) { + a, err := v.placedAsk(p, s, s.Ask, now) + if err != nil { + return nil, err + } + if s.Rerun { + markRerun(a, s.By) + } + return a, nil +} + +// markRerun marks a's body a rerun by caller. +func markRerun(a *Asked, caller string) { + switch b := a.Ask.(type) { + case nodebuildgo.StepAsk: + if b.Step != nil { + step := *b.Step + step.Rerun, step.By = true, caller + b.Step = &step + } + a.Ask = b + case builds.StepRun: + b.Rerun, b.By = true, caller + a.Ask = b + case wants.StepRun: + b.Rerun, b.By = true, caller + a.Ask = b + } +} + +// endUnanswered ends the running step of pipeline key asked under ask could not run, its ask not answered (H-A-235): every +// other mover's validate reading the ask with it, a composition its validate held ended not asked, the pipeline ended when +// nothing else is open, said and logged as any step's end. +func (h *Hearer) endUnanswered(key, ask string) error { + for { + p, rev, err := h.Store.Get(key) + if err != nil { + return nil + } + i := stepOfAsk(p, ask) + if i < 0 || !open(p.Steps[i]) { + return nil + } + now := h.now() + s := &p.Steps[i] + s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeCouldNotRun, UnansweredWhy(ask), &now + p.Unlogged = append(p.Unlogged, endEntry(key, *s, now)) + var readers []int + if s.Step == seat.StepResultStepValidate { + readers = sameValidate(&p, i, now) + _, notAsked := releaseCompositions(&p, now) + readers = append(readers, notAsked...) + } + asked, err := askApproval(key, &p, now, h.Asker) + if err != nil { + return err + } + ended := h.endPipeline(&p, now) + if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return fmt.Errorf("pipeline %s cannot be written: %w", key, err) + } + h.Failures.StepEnded(h.Store, p, *s) + for _, j := range append([]int{i}, readers...) { + if err := h.say(seat.EventStepEnded, p, &p.Steps[j]); err != nil { + return fmt.Errorf("pipeline %s's %s is written but not said: %w", key, seat.EventStepEnded, err) + } + } + h.appendUnlogged(key) + return h.said(p, asked, ended) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go new file mode 100644 index 00000000..c32e5c2a --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go @@ -0,0 +1,102 @@ +package pipeline + +import ( + "slices" + "strings" + "testing" + "time" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" +) + +// H-A-235, H-A-238: every ask but a test run's is bounded from its put until its holder's first word. A bound that passes asks +// once more under the ask's own id, its second bound armed; past the second the step ends could not run "its ask was not +// answered", the finding raised, and the pipeline ends; a first word deletes both bounds, and a late one clears the finding. +func TestAnAskNobodyAnswersIsAskedOnceMoreThenEndsItsStep(t *testing.T) { + v, store, said, asks, _, _ := verbsFixture() + var armed, unbound, raised, cleared []string + logs := []string{} + v.Hearer.Logf = func(f string, a ...any) { logs = append(logs, f) } + c := &AskClock{Bound: 10 * time.Minute, + Arm: func(key string, ttl time.Duration) error { armed = append(armed, key+" "+ttl.String()); return nil }, + Unbound: func(key string) error { unbound = append(unbound, key); return nil }, + Raise: func(s, ask string, _ time.Duration) error { raised = append(raised, s+" "+ask); return nil }, + Clear: func(s, ask string) error { cleared = append(cleared, s+" "+ask); return nil }} + v.Starter.Asks, v.Hearer.Asks = c, c + + // A test run's ask keeps §725's bounds; a nil clock arms none. + if err := c.arm(Asked{Seat: nodebuildgo.Seat, ID: "r1", Run: "run"}); err != nil || len(armed) != 0 { + t.Fatalf("a test run's ask armed %v (%v)", armed, err) + } + if err := (*AskClock)(nil).arm(Asked{Seat: nodebuildgo.Seat, ID: "x"}); err != nil { + t.Fatal(err) + } + + // Every step but app:program's lint has ended; that lint failed, and its rerun's ask is armed before it is put. + p, rev, _ := store.Get(pkey) + for i := range p.Steps { + if open(p.Steps[i]) { + p.Steps[i].Outcome = seat.StepResultOutcomePassed + } + } + p.Steps[stepIndex(p, "lint", "app:program")].Outcome = seat.StepResultOutcomeFailed + store.Update(pkey, p, rev) + r, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Step: "lint", Artifact: "app:program", Why: "retry"}) + if err != nil { + t.Fatal(err) + } + ask := r.Result.Ask + if want := []string{AskBoundKey(nodebuildgo.Seat, ask) + " 10m0s"}; !slices.Equal(armed, want) || len(asks.ids) != 1 { + t.Fatalf("armed %v asking %v, want %v before one ask", armed, asks.ids, want) + } + // The first bound passes: asked once more under its own id, a rerun still, the second bound armed and said. + if err := v.AskUnanswered(nodebuildgo.Seat, ask, false); err != nil { + t.Fatal(err) + } + if len(asks.ids) != 2 || asks.ids[1] != ask || armed[1] != AskAgainBoundKey(nodebuildgo.Seat, ask)+" 10m0s" { + t.Fatalf("asked %v, armed %v, want %s once more and its second bound", asks.ids, armed, ask) + } + if b, ok := asks.asks[1].(nodebuildgo.StepAsk); !ok || b.ID != ask || b.Step == nil || !b.Step.Rerun || b.Step.By != "operator" { + t.Errorf("the ask put again = %+v", asks.asks[1]) + } + if !slices.ContainsFunc(logs, func(l string) bool { return strings.Contains(l, "asked once more") }) { + t.Errorf("no journal line names the ask asked again: %v", logs) + } + if got, _, _ := store.Get(pkey); got.Steps[stepIndex(got, "lint", "app:program")].Outcome != seat.StepResultOutcomeRunning { + t.Fatal("the step ended at the first bound") + } + // The second bound passes: the step ends could not run, the finding is raised, the pipeline ends failed. + if err := v.AskUnanswered(nodebuildgo.Seat, ask, true); err != nil { + t.Fatal(err) + } + got, _, _ := store.Get(pkey) + s := got.Steps[stepIndex(got, "lint", "app:program")] + if s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+ask+" was not answered" || s.Ended == nil { + t.Errorf("the step = %+v", s) + } + if got.Outcome != seat.PipelineOutcomeFailed || len(asks.ids) != 2 { + t.Errorf("the pipeline is %s (%s), asked %d, want failed and nothing more asked", got.Outcome, got.Why, len(asks.ids)) + } + if !slices.Equal(raised, []string{nodebuildgo.Seat + " " + ask}) || !slices.Contains(*said, "pipeline-ended.main.5") { + t.Errorf("raised %v, said %v", raised, *said) + } + // A bound whose step ended asks nothing; a first word deletes both bounds; a late end clears the finding. + if err := v.AskUnanswered(nodebuildgo.Seat, ask, false); err != nil || len(asks.ids) != 2 { + t.Errorf("a bound of an ended step asked %v (%v)", asks.ids, err) + } + if err := c.Answered(nodebuildgo.Seat, ask); err != nil || !slices.Equal(unbound, []string{AskBoundKey(nodebuildgo.Seat, ask), AskAgainBoundKey(nodebuildgo.Seat, ask)}) { + t.Errorf("a first word deleted %v (%v)", unbound, err) + } + if err := v.Hearer.Taken(ask, "n1"); err != nil || !slices.Equal(cleared, []string{nodebuildgo.Seat + " " + ask}) { + t.Errorf("a late ask-taken cleared %v (%v)", cleared, err) + } + for _, k := range []string{AskBoundKey("mesh-wants", "a1"), AskAgainBoundKey("mesh-wants", "a1")} { + if s, a, again, ok := ParseAskBound(k); !ok || s != "mesh-wants" || a != "a1" || again != strings.HasPrefix(k, AskAgainBoundPrefix) { + t.Errorf("ParseAskBound(%s) = %s %s %v %v", k, s, a, again, ok) + } + } + if _, _, _, ok := ParseAskBound(DropBoundKey("node-build-go", "a1")); ok { + t.Error("a drop's bound read as an ask's") + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go b/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go index f3847b25..c95cba8b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go @@ -11,8 +11,8 @@ import ( ) // DropBound is how long a superseded step's drop may go unanswered before it is said (design 47 G A-222, A-77): five minutes, -// a key of the state `bounds` with its own time to live. An ask waiting for its holder has no such bound (§535, §647 as -// A-466y amends them): it is timed by §86's build-capacity finding and, for a test run, by run-untaken-bound (§725). +// a key of the state `bounds` with its own time to live. An ask waiting for its holder is bounded by ask-unanswered-bound +// (H-A-235, H-A-238; AskClock), a test run's by run-untaken-bound (§725). const DropBound = 5 * time.Minute // TakenSeat is a build seat whose holder says `ask-taken` when it takes an ask (design 47 G §602): where the runner hears it. diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index a7dbf327..ef891deb 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -387,17 +387,23 @@ func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Ar Module: m.Module, Pipeline: p.Pipeline, Revision: p.Revision, Source: p.Source, Step: step}} } -// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its -// holder with no bound of the runner's own (§535, §647 as A-466y amends them). -func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit string) bool) error { +// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. arm, when not nil, arms +// each ask's bound before it is put (H-A-235, H-A-238 (5)); an ask whose bound is not armed is not put. +func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit string) bool, arm func(Asked) error) error { var first error for _, a := range asks { accept, ok := acceptStep[a.Seat] if to, named := acceptRecovery[a.Seat]; named && recovery != nil && recovery(a.Commit) { accept = to } - err := fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat) - if ok { + var err error + switch { + case !ok: + err = fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat) + case arm != nil: + err = arm(a) + } + if err == nil { err = asker.AskStep(accept, a.ID, a.Ask) } if err != nil && first == nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 68dd3f35..1075dbab 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -183,6 +183,8 @@ type Starter struct { // Queue bounds the pipelines asked at once to pipelines-at-once, the others waiting in order (design 47 H §748); nil // places every start at once. Queue *Queue + // Asks bounds every ask but a test run's until its holder's first word (H-A-235, H-A-238); nil bounds none. + Asks *AskClock // RunTurn places a run on request whose turn in the queue came, from its candidate's newest revision (A-222); nil places // it at the runner's next start. RunTurn func(UnplacedRun) @@ -252,7 +254,7 @@ func (s *Starter) askSteps(key string, asks []Asked) error { var err error if len(asks) > 0 { s.carry(key, asks) - err = askAll(s.Steps, key, asks, func(commit string) bool { return s.recovered(key, commit) }) + err = askAll(s.Steps, key, asks, func(commit string) bool { return s.recovered(key, commit) }, s.Asks.arm) } s.sayStarted(key, placed) return err diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 78a4d3f8..5609652d 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -176,6 +176,9 @@ type Hearer struct { // Unplaced says why a mover of a pipeline has no step placed, for the why of its end (H-A-171; Starter.Unplaced): it // waits on its registration or on a publish; nil says only that it has no step placed. Unplaced func(p seat.Pipeline, module string) string + // Asks clears the finding of an ask that ended its step unanswered once it is answered after all (H-A-238 (2)); nil + // clears none. + Asks *AskClock // Rejoin keys a pipeline running again once its approval is answered and it still runs (H-A-106 (b); Starter.Rejoin); // nil keys none. Rejoin func(key string) @@ -249,6 +252,7 @@ func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, sh return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s of %s at %s (%s)", ErrRefused, ev.ID, ev.Step, ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Steps[i].Module, m.Commit, m.Fingerprint) case !open(p.Steps[i]): + h.Asks.answeredLate(p.Steps[i], h.logf) if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil { if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID { // The asking step's end was written and its run's was not (a write failed): the run ends now, heard again. diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go index 104c792b..a040e507 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go @@ -363,7 +363,7 @@ func TestARecoverysStepsAreAskedOnTheStreamDelivery(t *testing.T) { }{{true, "mesh.seat.node-build-go.accept.delivery.mesh-pipeline-runner," + wants.AcceptSubjectStep}, {false, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner," + wants.AcceptSubjectStep}} { var to []string asker := askerFunc(func(accept, _ string, _ any) error { to = append(to, accept); return nil }) - if err := askAll(asker, "k", asks, func(string) bool { return c.recovery }); err != nil || strings.Join(to, ",") != c.want { + if err := askAll(asker, "k", asks, func(string) bool { return c.recovery }, nil); err != nil || strings.Join(to, ",") != c.want { t.Errorf("recovery %v: asked on %v (%v), want %s", c.recovery, to, err, c.want) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go index 6d77cbda..bf535acf 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go @@ -34,6 +34,9 @@ func (h *Hearer) Taken(ask, node string) error { break } i := stepOfAsk(p, ask) + if i >= 0 && !open(p.Steps[i]) { + h.Asks.answeredLate(p.Steps[i], h.logf) + } if i < 0 || !open(p.Steps[i]) || p.Steps[i].Node == node { break } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 6ccd0f0b..7568e156 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -6,7 +6,6 @@ import ( "strings" "time" - builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" @@ -321,7 +320,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error } if ask != nil { v.Starter.carry(in.Pipeline, []Asked{*ask}) - if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }); err != nil { + if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm); err != nil { return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err) } } @@ -468,7 +467,7 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) break } v.Starter.carry(in.Pipeline, []Asked{*ask}) - if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }); err != nil { + if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm); err != nil { return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err) } } @@ -500,8 +499,24 @@ func (v Verbs) passedReader(reader RunReader, run string) bool { } // askOf is the ask to put on the serving seat for step s again: the one Place makes for it, under a new id (an ask put twice -// under one id is kept once), saying it is a rerun and by whom. +// under one id is kept once), saying it is a rerun and by whom. The validate ask is the pipeline's one, naming no mover +// (H-A-51): its rerun asks that one ask again; a composition's rerun builds the mover's composition ask (H-A-236). func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time.Time) (*Asked, error) { + id := AskID(now, p.Pipeline, s.Module, s.Artifact, s.Step) + if s.Seat == wants.Seat && s.Step == seat.StepResultStepValidate { + id = AskID(now, p.Pipeline, "", "", seat.StepResultStepValidate) + } + a, err := v.placedAsk(p, s, id, now) + if err != nil { + return nil, err + } + markRerun(a, caller) + return a, nil +} + +// placedAsk is the ask of step s under id: a validate or composition step of the seat mesh-wants built directly for its +// mover (H-A-236), any other the one Place makes for it. +func (v Verbs) placedAsk(p seat.Pipeline, s seat.StepResult, id string, now time.Time) (*Asked, error) { if v.Starter == nil || v.Starter.Registration == nil || v.Starter.Steps == nil || (v.View == nil && p.Run == "") { return nil, errors.New("the runner cannot ask a step: it holds no registration or no ask") } @@ -509,19 +524,13 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time if !ok { return nil, fmt.Errorf("%s moves no module %q", p.Pipeline, s.Module) } - // The validate ask is the pipeline's one, naming no mover (H-A-51): its rerun asks that one ask again; a composition's - // rerun builds the mover's composition ask directly, never looked up among the placed steps (H-A-236). - switch s.Step { - case seat.StepResultStepValidate, seat.StepResultStepComposition: - if s.Seat == wants.Seat { - id, step := AskID(now, p.Pipeline, "", "", seat.StepResultStepValidate), wants.StepRunStepValidate - if s.Step == seat.StepResultStepComposition { - id, step = AskID(now, p.Pipeline, m.Module, "", seat.StepResultStepComposition), wants.StepRunStepComposition - } - a := wantsAsk(p, m, id, step) - b := a.Ask.(wants.StepRun) - b.Rerun, b.By = true, caller - a.Ask = b + if s.Seat == wants.Seat { + switch s.Step { + case seat.StepResultStepValidate: + a := wantsAsk(p, m, id, wants.StepRunStepValidate) + return &a, nil + case seat.StepResultStepComposition: + a := wantsAsk(p, m, id, wants.StepRunStepComposition) return &a, nil } } @@ -549,21 +558,9 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time if a.ID != base { continue } - a.ID = AskID(now, p.Pipeline, m.Module, s.Artifact, s.Step) - switch b := a.Ask.(type) { - case nodebuildgo.StepAsk: - b.ID = a.ID - if b.Step != nil { - step := *b.Step - step.Rerun, step.By = true, caller - b.Step = &step - } - a.Ask = b - case builds.StepRun: - b.Rerun, b.By = true, caller - a.Ask = b - case wants.StepRun: - b.Rerun, b.By = true, caller + a.ID = id + if b, ok := a.Ask.(nodebuildgo.StepAsk); ok { + b.ID = id a.Ask = b } return &a, nil diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index e500b0a8..5f88bcf8 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -84,7 +84,7 @@ func TestManifest(t *testing.T) { "mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended", "operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended", "node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", - "node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-catalog.registration-changed", + "node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-wants.step-progress", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-catalog.version-released", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged", // A-186: the seat list its write verbs' callers are checked against, heard as it changes. "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", @@ -199,7 +199,7 @@ func runSettingsOf(raw []byte) bool { if json.Unmarshal(raw, &m) != nil { return false } - want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "widens-read-bound": "string", "revisions-kept": "number", + want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "ask-unanswered-bound": "string", "widens-read-bound": "string", "revisions-kept": "number", "pipelines-at-once": "number", "pipelines-waiting-cap": "number"} if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) || m.Config["pipelines-at-once"]["minimum"] != float64(1) || m.Config["pipelines-at-once"]["maximum"] != float64(8) || diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index 671a0b97..c0e1e96a 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -58,6 +58,7 @@ "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", + "mesh-wants.step-progress", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-catalog.version-released", @@ -151,6 +152,12 @@ "description": "how long a test run's ask may wait untaken after the run's start before the run ends could not run, a Go duration such as 24h (design 47 G section 725, A-466s, A-469x).", "trusted": false, "default": "24h" + }, + "ask-unanswered-bound": { + "type": "string", + "description": "how long an ask the runner puts on a seat, but a test run's, may go without its holder's first word (an ask-taken, a step progress or the step's end) before it is asked once more, and as long again before its step ends could not run, a Go duration such as 10m (design 47 H H-A-235, H-A-238).", + "trusted": false, + "default": "10m" } }, "state": [ @@ -278,6 +285,14 @@ "mode": "0600", "content": "${config:run-untaken-bound}" }, + { + "id": "ask-unanswered-bound", + "type": "file", + "path": "${dir:state}/ask-unanswered-bound", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:ask-unanswered-bound}" + }, { "id": "widens-read-bound", "type": "file", @@ -326,6 +341,7 @@ "MESH_RUNNER_WIDENS_READ_BOUND_FILE": "${dir:state}/widens-read-bound", "MESH_RUNNER_REVISIONS_KEPT_FILE": "${dir:state}/revisions-kept", "MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound", + "MESH_RUNNER_ASK_UNANSWERED_BOUND_FILE": "${dir:state}/ask-unanswered-bound", "MESH_RUNNER_PIPELINES_AT_ONCE_FILE": "${dir:state}/pipelines-at-once", "MESH_RUNNER_PIPELINES_WAITING_CAP_FILE": "${dir:state}/pipelines-waiting-cap" }, From 902947a044d8d7d23277adeca4eb50054360bfb6 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 11:42:43 +0100 Subject: [PATCH 3/8] Bound a silent seat's ask once; put it again when its holder starts (H-A-243) mesh-builds and mesh-wants say nothing before a step's end, so asking them again at the short bound would double a running build. Their ask has one bound, ask-silent-bound, and is put again when the node-engine says the holder running, since the holder takes asks on a plain subscription and loses those put while it is down. An ask asked again on a build seat carries a second id derived in the accepted form. Builds: design 47 H H-A-235, H-A-238, H-A-241, H-A-243 --- .../cmd/mesh-delivery-pipeline/askbounds.go | 50 ++++- .../cmd/mesh-delivery-pipeline/main.go | 11 +- .../queue_integration_test.go | 40 +++- .../testruns_integration_test.go | 25 +++ modules/mesh-delivery-pipeline/go.mod | 2 +- .../internal/pipeline/asks.go | 175 +++++++++++++++--- .../internal/pipeline/asks_test.go | 77 +++++++- .../internal/pipeline/steps.go | 3 +- .../mesh-delivery-pipeline/manifest_test.go | 4 +- modules/mesh-delivery-pipeline/module.json | 16 ++ 10 files changed, 347 insertions(+), 56 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go index b91e9069..94d7323e 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go @@ -9,6 +9,7 @@ import ( "github.com/nats-io/nats.go/jetstream" + nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" @@ -20,16 +21,28 @@ import ( // (design 47 H H-A-238 (1)). const envAskUnansweredBound = "MESH_RUNNER_ASK_UNANSWERED_BOUND_FILE" +// envAskSilentBound names the file of the setting ask-silent-bound (H-A-243 (2)). +const envAskSilentBound = "MESH_RUNNER_ASK_SILENT_BOUND_FILE" + // askUnansweredBound reads the setting ask-unanswered-bound at the start, a Go duration (H-A-235, H-A-238 (1)): its default // ten minutes when the file is missing or empty, an error naming the file for a content that is no positive duration. func askUnansweredBound(getenv func(string) string) (time.Duration, error) { - v, err := settingText(getenv(envAskUnansweredBound)) + return durationSetting(getenv, envAskUnansweredBound, pipeline.DefaultAskUnansweredBound) +} + +// askSilentBound reads the setting ask-silent-bound at the start, a Go duration, forty-five minutes by default (H-A-243 (2)). +func askSilentBound(getenv func(string) string) (time.Duration, error) { + return durationSetting(getenv, envAskSilentBound, pipeline.DefaultAskSilentBound) +} + +func durationSetting(getenv func(string) string, env string, def time.Duration) (time.Duration, error) { + v, err := settingText(getenv(env)) if err != nil || v == "" { - return pipeline.DefaultAskUnansweredBound, err + return def, err } d, err := time.ParseDuration(v) if err != nil || d <= 0 { - return 0, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(envAskUnansweredBound), v) + return 0, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(env), v) } return d, nil } @@ -37,8 +50,8 @@ func askUnansweredBound(getenv func(string) string) (time.Duration, error) { // askClock is the bound of every ask but a test run's over the state bounds (H-A-238 (5)): a key armed with its own time to // live before the ask is put, deleted at its holder's first word, its expiry heard by watchBounds; the finding call-unanswered, // scope call, id the ask, raised at the second bound and cleared when the ask is answered after all (H-A-238 (2)). -func askClock(ctx context.Context, bounds jetstream.KeyValue, bound time.Duration, w callFindings, module string, now func() time.Time) *pipeline.AskClock { - return &pipeline.AskClock{Bound: bound, +func askClock(ctx context.Context, bounds jetstream.KeyValue, bound, silent time.Duration, w callFindings, module string, now func() time.Time) *pipeline.AskClock { + return &pipeline.AskClock{Bound: bound, Silent: silent, Arm: func(key string, ttl time.Duration) error { _, err := bounds.Create(ctx, key, []byte(`{"ask":"`+key+`"}`), jetstream.KeyTTL(ttl)) if errors.Is(err, jetstream.ErrKeyExists) { @@ -66,7 +79,7 @@ func askClock(ctx context.Context, bounds jetstream.KeyValue, bound time.Duratio return w.Write(ctx, condseat.Finding{Scope: mcc.ScopeCall, ID: ask, Kind: mcc.KindCallUnanswered, Severity: condseat.FindingSeverityWarning, Summary: fmt.Sprintf("the ask %s of the seat %s is unanswered", ask, onSeat), Headline: "A step's ask is not answered", - Explanation: fmt.Sprintf("The runner put the ask %s on the seat %s, asked it once more, and heard no word of its holder within %s, so its step ended could not run and its pipeline went on (H-A-235).", ask, onSeat, within), + Explanation: fmt.Sprintf("The runner put the ask %s on the seat %s and heard no word of its holder about it within %s, so its step ended could not run and its pipeline went on (H-A-235, H-A-243).", ask, onSeat, within), Source: module + "." + mcc.KindCallUnanswered, Observer: module, At: now().UTC(), Subject: condseat.Subject{Scope: mcc.ScopeCall, ID: ask}}) }, Clear: func(_, ask string) error { @@ -99,6 +112,31 @@ func hearWantsProgress(hub handlers, c *pipeline.AskClock, logf func(string, ... }) } +// hearHolderStarted puts again the open asks of each silent seat whose holder the node-engine says running (H-A-243 (1)). +func hearHolderStarted(hub handlers, started func(module string) (int, error), logf func(string, ...any)) error { + return hub.Handle(nodeengine.SubjectRunningAll, func(m jetstream.Msg) { + var ev nodeengine.Running + if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Module == "" { + logf("%s is no running: %v", m.Subject(), err) + _ = m.Term() + return + } + if ev.Stopped { + _ = m.Ack() + return + } + n, err := started(ev.Module) + switch { + case errors.Is(err, pipeline.ErrHoldersNotRead): + _ = m.Nak() // heard again once the view is read + return + case err != nil: + logf("the asks of the seats %s holds, put again as it runs on %s: %d, and one is not: %v", ev.Module, ev.Node, n, err) + } + _ = m.Ack() + }) +} + // askBoundExpired takes the expiry of an ask's bound (H-A-235): the first asks once more, the second ends its step. func askBoundExpired(unanswered func(onSeat, ask string, again bool) error, logf func(string, ...any)) func(key string) bool { return func(key string) bool { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 378dc72d..f5f8af67 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -167,8 +167,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - logger.Printf("asks: each unanswered ask bound %s, asked once more, then its step ends could not run", askBound) - clock := askClock(ctx, bounds, askBound, rv.Findings, cred.Module, time.Now) + silentBound, err := askSilentBound(getenv) + if err != nil { + return err + } + logger.Printf("asks: an ask unanswered bound %s, asked once more, then its step ends could not run; an ask of a seat that says nothing before its end bound %s", askBound, silentBound) + clock := askClock(ctx, bounds, askBound, silentBound, rv.Findings, cred.Module, time.Now) starter.Asks, hearer.Asks = clock, clock starter.Supersede = func(newer seat.Pipeline) { hearer.SupersedeRunning(newer) } hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) } @@ -317,6 +321,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // since an ask's expired bound asks it once more through them (H-A-235). verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls) verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill} + if err := hearHolderStarted(heard, verbs.HolderStarted, logger.Printf); err != nil { + return err + } dropExpired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf) askExpired := askBoundExpired(verbs.AskUnanswered, logger.Printf) expired := func(key string) { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go index 5ad1481c..84a5e92a 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go @@ -107,6 +107,17 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { const router = "operator-router" answer("mesh.seat."+seatlist.HolderSeat+".call."+seatlist.HolderVerb+".*", map[string]any{"seats": []any{ map[string]any{"seat": oc.Seat, "holders": []any{map[string]any{"module": router}}}}}) + // Each mesh-builds step ask as its holder hears it, on its plain subscription. + var buildAsksMu sync.Mutex + buildAsks := map[string]int{} + if _, err := nc.Subscribe(builds.AcceptSubjectStep, func(m *nats.Msg) { + buildAsksMu.Lock() + buildAsks[m.Header.Get(nats.MsgIdHdr)]++ + buildAsksMu.Unlock() + }); err != nil { + t.Fatal(err) + } + buildAsked := func(id string) int { buildAsksMu.Lock(); defer buildAsksMu.Unlock(); return buildAsks[id] } records, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: router + "_" + oc.StateAsks, History: 1}) if err != nil { t.Fatal(err) @@ -154,6 +165,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { t.Fatal(err) } askBound := filepath.Join(dir, "ask-unanswered-bound") + silentBound := filepath.Join(dir, "ask-silent-bound") logs := &syncBuffer{} start := func() func() { runCtx, stop := context.WithCancel(ctx) @@ -167,6 +179,8 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { return waitingCap case envAskUnansweredBound: return askBound + case envAskSilentBound: + return silentBound } return "" }, logs) @@ -362,12 +376,16 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { } // At the runner's next start the waiting main.10 is placed from its own state, saying wait-ended, which clears - // pipelines-stalled; the start begins no pipeline of its own. That start's ask-unanswered-bound is one second. + // pipelines-stalled; the start begins no pipeline of its own. That start's ask-unanswered-bound is one second, its + // ask-silent-bound four. stopRunner() setMaxMsgSize(t, ctx, js, "KV_"+module+"_"+seat.StateQueue, -1) if err := os.WriteFile(askBound, []byte("1s\n"), 0o600); err != nil { t.Fatal(err) } + if err := os.WriteFile(silentBound, []byte("4s\n"), 0o600); err != nil { + t.Fatal(err) + } before := pipelineKeys(t, ctx, pipelines) stopRunner = start() outcome("main.10", seat.PipelineOutcomeRunning) @@ -377,19 +395,21 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { t.Fatalf("the start wrote pipelines %v, want %v", after, before) } - // H-A-235, H-A-238: main.10's build ask, which no holder answers, is asked once more when its bound's key expires, and - // at its second bound's expiry its step ends could not run, the pipeline ends failed and frees its turn, and the finding - // call-unanswered names the ask. + // H-A-243: main.10's build ask on mesh-builds, whose holder says nothing before its end, is never asked again for + // ask-unanswered-bound (a build running four of those bounds is not touched); its one bound, + // ask-silent-bound, ends its step could not run "its ask had no end", the pipeline ending failed, its turn freed and + // the finding call-unanswered naming the ask. build := stepFor(get("main.10"), "app", seat.StepResultStepBuild) - if build.Ask == "" { + if build.Ask == "" || build.Started == nil { t.Fatalf("main.10's build = %+v, want it asked", build) } - waitFor(t, ctx, logs, func() bool { - return strings.Contains(logs.String(), "the ask "+build.Ask+" of the build step of main.10 on mesh-builds had no word from its holder within 1s: asked once more") - }) outcome("main.10", seat.PipelineOutcomeFailed) - if s := stepFor(get("main.10"), "app", seat.StepResultStepBuild); s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build.Ask+" was not answered" { - t.Fatalf("main.10's build = %+v, want could not run, its ask not answered", s) + s := stepFor(get("main.10"), "app", seat.StepResultStepBuild) + if s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build.Ask+" had no end" || s.Ended == nil || s.Ended.Sub(*build.Started) < 4*time.Second { + t.Fatalf("main.10's build = %+v, want could not run, its ask with no end, at its silent bound of 4s", s) + } + if strings.Contains(logs.String(), ": asked once more as ") || buildAsked(pipeline.AgainID(build.Ask)) != 0 { + t.Fatalf("the silent build's ask was asked again for ask-unanswered-bound (put %d times, again %d)", buildAsked(build.Ask), buildAsked(pipeline.AgainID(build.Ask))) } waitFor(t, ctx, logs, func() bool { e, err := findings.Get(ctx, "call."+build.Ask+".call-unanswered") diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go index 182cb70f..1ad69414 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go @@ -16,6 +16,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" @@ -99,6 +100,19 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { if _, err := holders.Put(ctx, "node-build-go", []byte(`{"holders":[{"module":"mesh-build-go"}]}`)); err != nil { t.Fatal(err) } + // mesh-builds is held by the builder, which takes its asks on a plain subscription and says nothing before an end. + if _, err := holders.Put(ctx, builds.Seat, []byte(`{"holders":[{"module":"mesh-delivery-builder"}]}`)); err != nil { + t.Fatal(err) + } + var buildAsksMu sync.Mutex + buildAsks := map[string]int{} + if _, err := nc.Subscribe(builds.AcceptSubjectStep, func(m *nats.Msg) { + buildAsksMu.Lock() + buildAsks[m.Header.Get(nats.MsgIdHdr)]++ + buildAsksMu.Unlock() + }); err != nil { + t.Fatal(err) + } // The registrations: app tests its program and tools bundle from one directory, its unit step required and its // integration step optional; web is built by node-build-node. @@ -331,6 +345,17 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { const p3 = "main.1" takeIn("app", "c1", "src2:aa", 3, 1) placed(p3) // started by the take-in alone: the runner starts no pipeline by sweep (design 47 H §747) + // H-A-243 (1): the node-engine's running of the builder puts its open build ask again, under its own id: an ask put while + // it was down is lost; a running of a module holding no silent seat puts nothing again. + firstBuild := step(read(p3), "", seat.StepResultStepBuild).Ask + publish(nodeengine.SubjectRunning("n1", "mesh-build-go"), nodeengine.Running{Module: "mesh-build-go", Node: "n1", At: time.Now().UTC()}) + publish(nodeengine.SubjectRunning("n1", "mesh-delivery-builder"), nodeengine.Running{Module: "mesh-delivery-builder", Node: "n1", At: time.Now().UTC()}) + waitFor(t, ctx, logs, func() bool { + buildAsksMu.Lock() + n := buildAsks[firstBuild] + buildAsksMu.Unlock() + return n == 2 && strings.Count(logs.String(), "the ask "+firstBuild+" of the build step of "+p3+" is put again on mesh-builds") == 1 + }) passBuild(p3, "app", "c1", "src2:aa") unitRun, unit := runOfStep(p3, "app:program", "unit") integRun, integ := runOfStep(p3, "app:program", "integration") diff --git a/modules/mesh-delivery-pipeline/go.mod b/modules/mesh-delivery-pipeline/go.mod index 58a8de3f..5ea7ce85 100644 --- a/modules/mesh-delivery-pipeline/go.mod +++ b/modules/mesh-delivery-pipeline/go.mod @@ -10,7 +10,7 @@ require ( require ( git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect + git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/asks.go b/modules/mesh-delivery-pipeline/internal/pipeline/asks.go index 3f009af3..e1d1ee1d 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/asks.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/asks.go @@ -1,6 +1,8 @@ package pipeline import ( + "crypto/sha256" + "encoding/hex" "errors" "fmt" "strings" @@ -16,6 +18,24 @@ import ( // from an ask until its holder's first word. const DefaultAskUnansweredBound = 10 * time.Minute +// DefaultAskSilentBound is the setting ask-silent-bound's default (H-A-243 (2)): forty-five minutes from an ask on a seat whose +// holder says no word before the step's end (mesh-builds, mesh-wants) until that end. +const DefaultAskSilentBound = 45 * time.Minute + +// Silent says whether the holder of a seat the runner asks says nothing of an ask before its step's end: every seat but the +// build seats, which say ask-taken (H-A-241, H-A-243). +func Silent(onSeat string) bool { return !BoundSeat(onSeat) } + +// AgainID is the id an ask is put again under (H-A-243 (3)): the first id's time part and six hex digits of the sha256 of the +// first id and ".2", the form the holders accept, so no duplicate window is relied on. +func AgainID(id string) string { + if len(id) <= 6 { + return id + } + sum := sha256.Sum256([]byte(id + ".2")) + return id[:len(id)-6] + hex.EncodeToString(sum[:3]) +} + // The keys of the state bounds that time an ask the runner put (H-A-238 (5)): `ask..` armed before the ask is put, // `ask2..` armed when it is asked once more, each with the setting as its own time to live. const ( @@ -43,11 +63,16 @@ func ParseAskBound(key string) (onSeat, ask string, again, ok bool) { // UnansweredWhy is the why of a step whose ask was not answered within its two bounds (H-A-235). func UnansweredWhy(ask string) string { return "its ask " + ask + " was not answered" } +// SilentWhy is the why of a step on a silent seat whose ask had no end within ask-silent-bound (H-A-243 (2)). +func SilentWhy(ask string) string { return "its ask " + ask + " had no end" } + // AskClock bounds every ask the runner puts on a seat but a test run's, which keeps §725's bounds (H-A-235, H-A-238 (4)): // Arm writes a key of the state bounds with Bound as its own time to live, Unbound deletes one; Raise raises the finding // call-unanswered of an ask that ended its step, Clear clears it once the ask is answered after all (H-A-238 (2)). type AskClock struct { + // Bound is ask-unanswered-bound, for a seat that says ask-taken; Silent is ask-silent-bound, for a silent seat. Bound time.Duration + Silent time.Duration Arm func(key string, ttl time.Duration) error Unbound func(key string) error Raise func(onSeat, ask string, bound time.Duration) error @@ -59,12 +84,23 @@ func (c *AskClock) arm(a Asked) error { if c == nil || c.Arm == nil || a.Run != "" { return nil } - if err := c.Arm(AskBoundKey(a.Seat, a.ID), c.bound()); err != nil { + if err := c.Arm(AskBoundKey(a.Seat, a.ID), c.boundOf(a.Seat)); err != nil { return fmt.Errorf("the bound of ask %s is not armed: %w", a.ID, err) } return nil } +// boundOf is the bound of an ask on onSeat: ask-silent-bound for a silent seat (H-A-243 (2)), else ask-unanswered-bound. +func (c *AskClock) boundOf(onSeat string) time.Duration { + if !Silent(onSeat) { + return c.bound() + } + if c.Silent > 0 { + return c.Silent + } + return DefaultAskSilentBound +} + func (c *AskClock) bound() time.Duration { if c.Bound > 0 { return c.Bound @@ -72,18 +108,21 @@ func (c *AskClock) bound() time.Duration { return DefaultAskUnansweredBound } -// Answered deletes both bounds of the ask on onSeat, its holder's first word heard: an ask-taken, a step progress or the -// step's end (H-A-238 (3)). +// Answered deletes the bounds of the ask heard on onSeat, its holder's first word: an ask-taken, a step progress or, for a +// silent seat, the step's end (H-A-238 (3), H-A-241). Heard under a first id, the second bound of the ask put again under +// AgainID goes too. func (c *AskClock) Answered(onSeat, ask string) error { if c == nil || c.Unbound == nil || ask == "" { return nil } - return errors.Join(c.Unbound(AskBoundKey(onSeat, ask)), c.Unbound(AskAgainBoundKey(onSeat, ask))) + return errors.Join(c.Unbound(AskBoundKey(onSeat, ask)), c.Unbound(AskAgainBoundKey(onSeat, ask)), + c.Unbound(AskAgainBoundKey(onSeat, AgainID(ask)))) } -// answeredLate clears the finding of the ask of step s, which ended unanswered, its holder heard after all (H-A-238 (2)). +// answeredLate clears the finding of the ask of step s, which ended unanswered or with no end, its holder heard after all +// (H-A-238 (2)). func (c *AskClock) answeredLate(s seat.StepResult, logf func(string, ...any)) { - if c == nil || c.Clear == nil || s.Ask == "" || s.Why != UnansweredWhy(s.Ask) { + if c == nil || c.Clear == nil || s.Ask == "" || (s.Why != UnansweredWhy(s.Ask) && s.Why != SilentWhy(s.Ask)) { return } if err := c.Clear(s.Seat, s.Ask); err != nil { @@ -91,10 +130,12 @@ func (c *AskClock) answeredLate(s seat.StepResult, logf func(string, ...any)) { } } -// AskUnanswered takes the expiry of a bound of the ask on onSeat (H-A-235): at the first the ask is put once more under its -// own id, said in the journal, its second bound armed; at the second the step ends could not run "its ask was not -// answered", the finding call-unanswered raised, so the pipeline ends and its turn frees. A bound whose ask no running step -// names any longer asks nothing. +// AskUnanswered takes the expiry of a bound of the ask on onSeat (H-A-235, H-A-243). On a silent seat its one bound, +// ask-silent-bound, ends the step could not run "its ask had no end", never asking again (that would double a running +// build). On a seat that says ask-taken the first bound puts the ask once more under AgainID, said in the journal, its +// second bound armed under that id; the second ends the step could not run "its ask was not answered". An ended step +// raises the finding call-unanswered, so the pipeline ends and its turn frees. A bound whose ask no running step names any +// longer asks nothing. func (v Verbs) AskUnanswered(onSeat, ask string, again bool) error { h, c := v.Hearer, v.Starter.Asks if c == nil { @@ -105,26 +146,30 @@ func (v Verbs) AskUnanswered(onSeat, ask string, again bool) error { return err } s := p.Steps[i] - if again { - h.logf("the ask %s of the %s step of %s on %s had no word from its holder within its two bounds: the step ends could not run (H-A-235)", - ask, s.Step, key, onSeat) - if err := h.endUnanswered(key, ask); err != nil { + if Silent(onSeat) || again { + why, within := UnansweredWhy(s.Ask), 2*c.bound() + if Silent(onSeat) { + why, within = SilentWhy(s.Ask), c.boundOf(onSeat) + } + h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s: the step ends could not run (H-A-235, H-A-243)", + s.Ask, s.Step, key, onSeat, within) + if err := h.endUnanswered(key, s.Ask, why); err != nil { return err } if c.Raise != nil { - return c.Raise(onSeat, ask, 2*c.bound()) + return c.Raise(onSeat, s.Ask, within) } return nil } - a, err := v.askAgain(p, s, h.now()) + a, err := v.askAgain(p, s, AgainID(s.Ask), h.now()) if err != nil { - h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s and cannot be put again: %v", ask, s.Step, key, onSeat, c.bound(), err) + h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s and cannot be put again: %v", s.Ask, s.Step, key, onSeat, c.bound(), err) } else { - h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s: asked once more (H-A-235)", ask, s.Step, key, onSeat, c.bound()) + h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s: asked once more as %s (H-A-235, H-A-243)", s.Ask, s.Step, key, onSeat, c.bound(), a.ID) } if c.Arm != nil { - if aerr := c.Arm(AskAgainBoundKey(onSeat, ask), c.bound()); aerr != nil { - return fmt.Errorf("the second bound of ask %s is not armed, so it is not asked again: %w", ask, aerr) + if aerr := c.Arm(AskAgainBoundKey(onSeat, AgainID(s.Ask)), c.bound()); aerr != nil { + return fmt.Errorf("the second bound of ask %s is not armed, so it is not asked again: %w", s.Ask, aerr) } } if err != nil { @@ -134,7 +179,83 @@ func (v Verbs) AskUnanswered(onSeat, ask string, again bool) error { return askAll(v.Starter.Steps, key, []Asked{*a}, func(commit string) bool { return v.Starter.recovered(key, commit) }, nil) } -// runningAsk is the pipeline and the index of its running step asked under ask of onSeat; an empty key when none is. +// HolderStarted puts again every open ask of each silent seat module holds (H-A-243 (1)): its holder takes its asks on a plain +// subscription, so an ask put while it was down is lost; heard on the node-engine's running of module. Each ask is put under +// its own id, a rerun's marks kept, said in the journal; its bound stands as armed. +func (v Verbs) HolderStarted(module string) (int, error) { + st := v.Starter + if st == nil || st.Holders == nil { + return 0, nil + } + var silent []string + for onSeat := range acceptStep { + if !Silent(onSeat) { + continue + } + holders, read := st.Holders(onSeat) + if !read { + return 0, ErrHoldersNotRead + } + if containsString(holders, module) { + silent = append(silent, onSeat) + } + } + if len(silent) == 0 { + return 0, nil + } + pkeys, err := allKeys(v.Hearer.Store) + if err != nil { + return 0, fmt.Errorf("the pipelines are not listed: %w", err) + } + now, asked := v.Hearer.now(), 0 + var first error + for _, key := range pkeys { + p, _, err := v.Hearer.Store.Get(key) + if err != nil || p.Outcome != seat.PipelineOutcomeRunning { + continue + } + seen := map[string]bool{} + var due []Asked + for _, s := range p.Steps { + if !open(s) || s.Ask == "" || seen[s.Ask] || !containsString(silent, s.Seat) { + continue + } + seen[s.Ask] = true + a, err := v.askAgain(p, s, s.Ask, now) + if err != nil { + v.Hearer.logf("the ask %s of the %s step of %s on %s is not put again as %s starts: %v", s.Ask, s.Step, key, s.Seat, module, err) + continue + } + v.Hearer.logf("the ask %s of the %s step of %s is put again on %s: its holder %s started, and takes no ask put while it was down (H-A-243)", + s.Ask, s.Step, key, s.Seat, module) + due = append(due, *a) + } + if len(due) == 0 { + continue + } + st.carry(key, due) + if err := askAll(st.Steps, key, due, func(commit string) bool { return st.recovered(key, commit) }, nil); err != nil && first == nil { + first = err + } + asked += len(due) + } + return asked, first +} + +// ErrHoldersNotRead is HolderStarted's answer before the seat-holders view's first view is read: the running is heard again. +var ErrHoldersNotRead = errors.New("the seat-holders view is not read yet") + +func containsString(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +// runningAsk is the pipeline and the index of its running step asked under ask, or put again under it, of onSeat; an empty +// key when none is. func (v Verbs) runningAsk(onSeat, ask string) (string, seat.Pipeline, int, error) { pkeys, err := allKeys(v.Hearer.Store) if err != nil { @@ -152,9 +273,9 @@ func (v Verbs) runningAsk(onSeat, ask string) (string, seat.Pipeline, int, error return "", seat.Pipeline{}, -1, nil } -// askAgain is the ask of running step s as it was put: its own id, a rerun's marks kept. -func (v Verbs) askAgain(p seat.Pipeline, s seat.StepResult, now time.Time) (*Asked, error) { - a, err := v.placedAsk(p, s, s.Ask, now) +// askAgain is the ask of running step s as it was put, under id: a rerun's marks kept. +func (v Verbs) askAgain(p seat.Pipeline, s seat.StepResult, id string, now time.Time) (*Asked, error) { + a, err := v.placedAsk(p, s, id, now) if err != nil { return nil, err } @@ -183,10 +304,10 @@ func markRerun(a *Asked, caller string) { } } -// endUnanswered ends the running step of pipeline key asked under ask could not run, its ask not answered (H-A-235): every +// endUnanswered ends the running step of pipeline key asked under ask could not run, saying why (H-A-235, H-A-243): every // other mover's validate reading the ask with it, a composition its validate held ended not asked, the pipeline ended when // nothing else is open, said and logged as any step's end. -func (h *Hearer) endUnanswered(key, ask string) error { +func (h *Hearer) endUnanswered(key, ask, why string) error { for { p, rev, err := h.Store.Get(key) if err != nil { @@ -198,7 +319,7 @@ func (h *Hearer) endUnanswered(key, ask string) error { } now := h.now() s := &p.Steps[i] - s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeCouldNotRun, UnansweredWhy(ask), &now + s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeCouldNotRun, why, &now p.Unlogged = append(p.Unlogged, endEntry(key, *s, now)) var readers []int if s.Step == seat.StepResultStepValidate { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go index c32e5c2a..3d345451 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go @@ -6,8 +6,10 @@ import ( "testing" "time" + builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" ) // H-A-235, H-A-238: every ask but a test run's is bounded from its put until its holder's first word. A bound that passes asks @@ -50,14 +52,19 @@ func TestAnAskNobodyAnswersIsAskedOnceMoreThenEndsItsStep(t *testing.T) { if want := []string{AskBoundKey(nodebuildgo.Seat, ask) + " 10m0s"}; !slices.Equal(armed, want) || len(asks.ids) != 1 { t.Fatalf("armed %v asking %v, want %v before one ask", armed, asks.ids, want) } - // The first bound passes: asked once more under its own id, a rerun still, the second bound armed and said. + // The first bound passes: asked once more under the id derived from the first, in the form the holders accept, a rerun + // still, the second bound armed under it and said (H-A-243 (3)). + again := AgainID(ask) + if err := sdkstep.CheckAskID(again); err != nil || again == ask || again[:len(again)-6] != ask[:len(ask)-6] { + t.Fatalf("AgainID(%s) = %s (%v), want the first's time part and other hex digits in the ask id form", ask, again, err) + } if err := v.AskUnanswered(nodebuildgo.Seat, ask, false); err != nil { t.Fatal(err) } - if len(asks.ids) != 2 || asks.ids[1] != ask || armed[1] != AskAgainBoundKey(nodebuildgo.Seat, ask)+" 10m0s" { - t.Fatalf("asked %v, armed %v, want %s once more and its second bound", asks.ids, armed, ask) + if len(asks.ids) != 2 || asks.ids[1] != again || armed[1] != AskAgainBoundKey(nodebuildgo.Seat, again)+" 10m0s" { + t.Fatalf("asked %v, armed %v, want %s once more and its second bound", asks.ids, armed, again) } - if b, ok := asks.asks[1].(nodebuildgo.StepAsk); !ok || b.ID != ask || b.Step == nil || !b.Step.Rerun || b.Step.By != "operator" { + if b, ok := asks.asks[1].(nodebuildgo.StepAsk); !ok || b.ID != again || b.Step == nil || !b.Step.Rerun || b.Step.By != "operator" { t.Errorf("the ask put again = %+v", asks.asks[1]) } if !slices.ContainsFunc(logs, func(l string) bool { return strings.Contains(l, "asked once more") }) { @@ -67,7 +74,7 @@ func TestAnAskNobodyAnswersIsAskedOnceMoreThenEndsItsStep(t *testing.T) { t.Fatal("the step ended at the first bound") } // The second bound passes: the step ends could not run, the finding is raised, the pipeline ends failed. - if err := v.AskUnanswered(nodebuildgo.Seat, ask, true); err != nil { + if err := v.AskUnanswered(nodebuildgo.Seat, again, true); err != nil { t.Fatal(err) } got, _, _ := store.Get(pkey) @@ -85,10 +92,10 @@ func TestAnAskNobodyAnswersIsAskedOnceMoreThenEndsItsStep(t *testing.T) { if err := v.AskUnanswered(nodebuildgo.Seat, ask, false); err != nil || len(asks.ids) != 2 { t.Errorf("a bound of an ended step asked %v (%v)", asks.ids, err) } - if err := c.Answered(nodebuildgo.Seat, ask); err != nil || !slices.Equal(unbound, []string{AskBoundKey(nodebuildgo.Seat, ask), AskAgainBoundKey(nodebuildgo.Seat, ask)}) { + if err := c.Answered(nodebuildgo.Seat, ask); err != nil || !slices.Equal(unbound, []string{AskBoundKey(nodebuildgo.Seat, ask), AskAgainBoundKey(nodebuildgo.Seat, ask), AskAgainBoundKey(nodebuildgo.Seat, again)}) { t.Errorf("a first word deleted %v (%v)", unbound, err) } - if err := v.Hearer.Taken(ask, "n1"); err != nil || !slices.Equal(cleared, []string{nodebuildgo.Seat + " " + ask}) { + if err := v.Hearer.Taken(again, "n1"); err != nil || !slices.Equal(cleared, []string{nodebuildgo.Seat + " " + ask}) { t.Errorf("a late ask-taken cleared %v (%v)", cleared, err) } for _, k := range []string{AskBoundKey("mesh-wants", "a1"), AskAgainBoundKey("mesh-wants", "a1")} { @@ -100,3 +107,59 @@ func TestAnAskNobodyAnswersIsAskedOnceMoreThenEndsItsStep(t *testing.T) { t.Error("a drop's bound read as an ask's") } } + +// H-A-243: an ask on a seat whose holder says nothing before the step's end (mesh-builds) is never asked again for a bound: +// its one bound, ask-silent-bound, ends the step could not run "its ask had no end", the finding raised. The node-engine's +// running of the seat's holder puts every open ask of the seat again under its own id, a module holding no such seat none. +func TestASilentSeatsAskIsPutAgainWhenItsHolderStartsAndEndsAtItsSilentBound(t *testing.T) { + v, store, _, asks, _, _ := verbsFixture() + var armed, raised []string + c := &AskClock{Bound: time.Minute, Silent: 45 * time.Minute, + Arm: func(key string, ttl time.Duration) error { armed = append(armed, key+" "+ttl.String()); return nil }, + Raise: func(s, ask string, within time.Duration) error { + raised = append(raised, s+" "+ask+" "+within.String()) + return nil + }} + v.Starter.Asks, v.Hearer.Asks = c, c + v.Starter.Holders = func(s string) ([]string, bool) { + return map[string][]string{builds.Seat: {"mesh-delivery-builder"}}[s], true + } + p, rev, _ := store.Get(pkey) + // Every step has ended but app's build, asked of mesh-builds. + b := stepIndex(p, seat.StepResultStepBuild, "") + for i := range p.Steps { + if open(p.Steps[i]) && i != b { + p.Steps[i].Outcome = seat.StepResultOutcomePassed + } + } + if p.Steps[b].Seat != builds.Seat || !open(p.Steps[b]) { + t.Fatalf("the build step = %+v, want running on mesh-builds", p.Steps[b]) + } + store.Update(pkey, p, rev) + build := p.Steps[b].Ask + if err := c.arm(Asked{Seat: builds.Seat, ID: build}); err != nil || !slices.Equal(armed, []string{AskBoundKey(builds.Seat, build) + " 45m0s"}) { + t.Fatalf("armed %v (%v), want the silent bound", armed, err) + } + // A module holding no silent seat puts nothing again; the builder's running puts the build again under its own id. + if n, err := v.HolderStarted("mesh-build-go"); err != nil || n != 0 || len(asks.ids) != 0 { + t.Fatalf("another holder's start put %d (%v)", n, err) + } + if n, err := v.HolderStarted("mesh-delivery-builder"); err != nil || n != 1 || len(asks.ids) != 1 || asks.ids[0] != build { + t.Fatalf("the builder's start put %d asks %v (%v), want %s again", n, asks.ids, err, build) + } + if _, ok := asks.asks[0].(builds.StepRun); !ok || asks.at[0] != builds.AcceptSubjectStep { + t.Errorf("the ask put again = %+v on %s", asks.asks[0], asks.at[0]) + } + // The silent bound passes: no ask again, the step ends could not run, the finding is raised, the pipeline ends. + if err := v.AskUnanswered(builds.Seat, build, false); err != nil { + t.Fatal(err) + } + got, _, _ := store.Get(pkey) + s := got.Steps[b] + if len(asks.ids) != 1 || s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build+" had no end" || got.Outcome != seat.PipelineOutcomeFailed { + t.Errorf("asked %v; the build %+v; the pipeline %s", asks.ids, s, got.Outcome) + } + if !slices.Equal(raised, []string{builds.Seat + " " + build + " 45m0s"}) { + t.Errorf("raised %v", raised) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 5609652d..d9d93793 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -589,7 +589,8 @@ func endOutcome(o string) bool { func stepOfAsk(p seat.Pipeline, id string) int { for i, s := range p.Steps { - if id != "" && s.Ask == id { + // An ask put again is put under AgainID of the step's ask (H-A-243 (3)): its holder's answer under either is the step's. + if id != "" && s.Ask != "" && (s.Ask == id || AgainID(s.Ask) == id) { return i } } diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 5f88bcf8..7de59100 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -84,7 +84,7 @@ func TestManifest(t *testing.T) { "mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended", "operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended", "node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", - "node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-wants.step-progress", "mesh-catalog.registration-changed", + "node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-wants.step-progress", "node-engine.running", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-catalog.version-released", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged", // A-186: the seat list its write verbs' callers are checked against, heard as it changes. "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", @@ -199,7 +199,7 @@ func runSettingsOf(raw []byte) bool { if json.Unmarshal(raw, &m) != nil { return false } - want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "ask-unanswered-bound": "string", "widens-read-bound": "string", "revisions-kept": "number", + want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "ask-unanswered-bound": "string", "ask-silent-bound": "string", "widens-read-bound": "string", "revisions-kept": "number", "pipelines-at-once": "number", "pipelines-waiting-cap": "number"} if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) || m.Config["pipelines-at-once"]["minimum"] != float64(1) || m.Config["pipelines-at-once"]["maximum"] != float64(8) || diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index c0e1e96a..d25b519d 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -59,6 +59,7 @@ "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-wants.step-progress", + "node-engine.running", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-catalog.version-released", @@ -158,6 +159,12 @@ "description": "how long an ask the runner puts on a seat, but a test run's, may go without its holder's first word (an ask-taken, a step progress or the step's end) before it is asked once more, and as long again before its step ends could not run, a Go duration such as 10m (design 47 H H-A-235, H-A-238).", "trusted": false, "default": "10m" + }, + "ask-silent-bound": { + "type": "string", + "description": "how long an ask the runner puts on a seat whose holder says nothing before the step's end (mesh-builds, mesh-wants) may go without that end before its step ends could not run, a Go duration such as 45m (design 47 H H-A-243).", + "trusted": false, + "default": "45m" } }, "state": [ @@ -293,6 +300,14 @@ "mode": "0600", "content": "${config:ask-unanswered-bound}" }, + { + "id": "ask-silent-bound", + "type": "file", + "path": "${dir:state}/ask-silent-bound", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:ask-silent-bound}" + }, { "id": "widens-read-bound", "type": "file", @@ -342,6 +357,7 @@ "MESH_RUNNER_REVISIONS_KEPT_FILE": "${dir:state}/revisions-kept", "MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound", "MESH_RUNNER_ASK_UNANSWERED_BOUND_FILE": "${dir:state}/ask-unanswered-bound", + "MESH_RUNNER_ASK_SILENT_BOUND_FILE": "${dir:state}/ask-silent-bound", "MESH_RUNNER_PIPELINES_AT_ONCE_FILE": "${dir:state}/pipelines-at-once", "MESH_RUNNER_PIPELINES_WAITING_CAP_FILE": "${dir:state}/pipelines-waiting-cap" }, From 5f7137e676e150f0bb467e7369b261da0fffff20 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 11:44:49 +0100 Subject: [PATCH 4/8] A moved mover leaves an older waiting pipeline whenever a newer one starts (H-A-244) Only a newer pipeline that waited took movers out of older waiting ones; one placed at once left them, so main.84 was placed nine hours later with a mover a newer revision had moved and built a stale fingerprint. Builds: design 47 H H-A-223, H-A-244 (1) --- .../internal/pipeline/queue_test.go | 46 +++++++++++++++++++ .../internal/pipeline/start.go | 7 +-- .../internal/pipeline/supersede.go | 18 ++++++-- 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go index 54249bca..23b84166 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go @@ -176,6 +176,52 @@ func TestThePipelinesWaitTheirTurn(t *testing.T) { } } +// H-A-244 (1), H-A-223: a mover a newer revision moves leaves an older waiting pipeline also when the newer one is placed at +// once, not only when it waits: the waiting pipeline keeps the movers still its own, and with none left ends superseded, +// never placed. +func TestANewerPlacedPipelineSupersedesAnOlderWaitingOne(t *testing.T) { + s, mem, q, said, _, _, view, _ := queueFixture() + take := takeIn(t, s, view) + take(4, "app") + take(5, "lib", "tool") + if p, _, _ := mem.Get("main.5"); p.Outcome != seat.PipelineOutcomeWaiting { + t.Fatalf("main.5 = %s, want waiting", p.Outcome) + } + s.Queue.AtOnce = 3 + take(6, "lib") + if p, _, _ := mem.Get("main.6"); p.Outcome != seat.PipelineOutcomeRunning { + t.Fatalf("main.6 = %s, want placed at once", p.Outcome) + } + p5, _, _ := mem.Get("main.5") + if p5.Outcome != seat.PipelineOutcomeWaiting || len(p5.Movers) != 1 || p5.Movers[0].Module != "tool" || !slices.Equal(p5.Superseded, []string{"lib"}) { + t.Fatalf("main.5 = %s moving %+v, superseded %v; want waiting with tool alone, lib superseded", p5.Outcome, p5.Movers, p5.Superseded) + } + // At its turn it is placed with tool alone. + p4, rev, _ := mem.Get("main.4") + p4.Outcome = seat.PipelineOutcomePassed + mem.Update("main.4", p4, rev) + if _, err := s.Turn("main.4", view); err != nil { + t.Fatal(err) + } + if p, _, _ := mem.Get("main.5"); p.Outcome != seat.PipelineOutcomeRunning || len(stepsOf(p, "lib")) != 0 || len(stepsOf(p, "tool")) == 0 { + t.Fatalf("main.5 at its turn = %s with steps %+v, want running with tool's steps alone", p.Outcome, p.Steps) + } + + // A waiting pipeline whose one mover a newer placed pipeline moves ends superseded, never placed. + s, mem, q, said, _, _, view, _ = queueFixture() + take = takeIn(t, s, view) + take(4, "app") + take(5, "tool") + s.Queue.AtOnce = 3 + take(7, "tool") + if p, _, _ := mem.Get("main.5"); p.Outcome != seat.PipelineOutcomeSuperseded || len(p.Steps) != 0 || p.Ended == nil { + t.Errorf("main.5 = %s with %d steps, want superseded and never placed", p.Outcome, len(p.Steps)) + } + if _, ok := q["waiting.00000000000000000001"]; ok || !slices.Contains(*said, "pipeline-ended.main.5") { + t.Errorf("the queue %v, said %v; want main.5's waiting key gone and its pipeline-ended said", q, *said) + } +} + // R13/AC1 (design 47 H §748, ADR 0682 §7): a newer waiting pipeline of the same branch takes the movers it also moves out of // an older waiting one, which ends superseded, saying wait-ended and pipeline-ended, once no mover is left; a running // pipeline moving the same module runs to its end. diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 1075dbab..17168081 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -547,11 +547,12 @@ func (s *Starter) Start(b Begin) (bool, error) { if s.Supersede != nil { s.Supersede(p) } + // ... and every older waiting one, whether this one waits or is placed at once (H-A-244 (1)). + if _, err := s.supersedeWaiting(p); err != nil { + s.logf("the older waiting pipelines of %s are not superseded by %s: %v", p.Branch, key, err) + } if !place { s.sayQueueWait(seat.EventWaitHeld, p, now, "") - if _, err := s.supersedeWaiting(p); err != nil { - s.logf("the older waiting pipelines of %s are not superseded by %s: %v", p.Branch, key, err) - } s.Queue.overCap(key, s.logf) return true, nil } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go index bf535acf..0f2340ad 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go @@ -166,10 +166,11 @@ func (h *Hearer) supersedeRunningOne(key string, newer seat.Pipeline) bool { // supersedeWaiting takes, in each older waiting pipeline of newer's branch, the movers newer also moves out of it (design 47 H // §748, ADR 0682 §7): a waiting pipeline left with no mover ends superseded, its waiting key deleted, saying wait-ended and its -// pipeline-ended. newer is a pipeline just written waiting; a running pipeline's movers leave it by SupersedeRunning -// (H-A-223). It answers the pipelines it ended. +// pipeline-ended. newer is a pipeline just written, waiting or placed: a mover a newer revision moves leaves every pipeline not +// ended, a waiting one as a running one (H-A-223, H-A-244 (1)), so a waiting pipeline is placed with only the movers still its +// own; a running pipeline's movers leave it by SupersedeRunning. It answers the pipelines it ended. func (s *Starter) supersedeWaiting(newer seat.Pipeline) ([]string, error) { - if s.Queue == nil || newer.Outcome != seat.PipelineOutcomeWaiting { + if s.Queue == nil || (newer.Outcome != seat.PipelineOutcomeWaiting && newer.Outcome != seat.PipelineOutcomeRunning) { return nil, nil } wkeys, pipelines, err := s.Queue.Waiting() @@ -196,7 +197,7 @@ func (s *Starter) supersedeWaiting(newer seat.Pipeline) ([]string, error) { func (s *Starter) supersedeOne(wkey, key string, newer seat.Pipeline) (bool, error) { for { p, rev, err := s.Store.Get(key) - if err != nil || p.Outcome != seat.PipelineOutcomeWaiting || p.Branch != newer.Branch || p.Seq >= newer.Seq { + if err != nil || p.Outcome != seat.PipelineOutcomeWaiting || p.Branch != newer.Branch || !olderWaiting(p, newer) { return false, nil } left := slices.DeleteFunc(slices.Clone(p.Movers), func(m seat.Mover) bool { return Moves(newer, m.Module) }) @@ -235,6 +236,15 @@ func (s *Starter) supersedeOne(wkey, key string, newer seat.Pipeline) (bool, err } } +// olderWaiting says whether the waiting pipeline p is older than newer: by its sequence in the queue when newer waits too, +// by its catalog revision when newer was placed at once (H-A-244 (1)); a run on request's pipelines are never compared. +func olderWaiting(p, newer seat.Pipeline) bool { + if newer.Outcome == seat.PipelineOutcomeWaiting { + return p.Seq < newer.Seq + } + return p.Run == "" && newer.Run == "" && p.Revision < newer.Revision +} + // supersededWhy writes, into mover module's build steps, that a dependency it builds on was taken out of p by a newer // pipeline, so it builds against the newest published package (design 47 H §748; H-A-53, H-A-107). func supersededWhy(p seat.Pipeline, module string, entries map[string]catalog.CatalogEntry, steps []seat.StepResult) { From c9515da86fa92b41f00fef9a764d822088ff158f Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 11:49:53 +0100 Subject: [PATCH 5/8] Take a validate end by its ask alone, ending every mover's validate (H-A-252) The pipeline's one validate ask names no mover, but its end names the mover whose body the holder took; the runner compared that mover with the first step carrying the ask and dropped a passed validate, so the pipeline held its slot (main.88). Builds: design 47 H H-A-236, H-A-252 --- .../internal/pipeline/asks_test.go | 32 +++++++++++++++++ .../internal/pipeline/steps.go | 34 ++++++++++++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go index 3d345451..1395bdff 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go @@ -8,6 +8,7 @@ import ( builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" ) @@ -163,3 +164,34 @@ func TestASilentSeatsAskIsPutAgainWhenItsHolderStartsAndEndsAtItsSilentBound(t * t.Errorf("raised %v", raised) } } + +// H-A-252: the pipeline's one validate ask names no mover (H-A-51), and its holder's end names the mover whose body it took: +// it is matched by its ask alone and ends the validate step of every mover under it, whichever mover it names. +func TestAValidateEndNamingOneMoverEndsEveryMoversValidate(t *testing.T) { + h, store, _, _, _, _ := fixture() + p, rev, _ := store.Get(pkey) + p.Movers = []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}, {Module: "lib", Commit: "c2", Fingerprint: "src2:bb"}} + p.Steps = []seat.StepResult{ + {Module: "app", Step: seat.StepResultStepValidate, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "v1", Seat: wants.Seat}, + {Module: "lib", Step: seat.StepResultStepValidate, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "v1", Seat: wants.Seat}} + store.Update(pkey, p, rev) + var bs StepSeat + for _, x := range StepSeats { + if x.Seat == wants.Seat { + bs = x + } + } + ev := StepEnded{ID: "v1", Node: "n1", Pipeline: pkey, Module: "lib", Commit: "c2", Fingerprint: "src2:bb", Step: seat.StepResultStepValidate, Outcome: "passed"} + if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", ev); err != nil { + t.Fatal(err) + } + got, _, _ := store.Get(pkey) + for _, s := range got.Steps { + if s.Outcome != seat.StepResultOutcomePassed { + t.Errorf("%s's validate = %s, want passed by the one end", s.Module, s.Outcome) + } + } + if got.Outcome != seat.PipelineOutcomePassed { + t.Errorf("the pipeline = %s (%s), want passed", got.Outcome, got.Why) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index d9d93793..5b1b8edf 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -239,6 +239,12 @@ func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, sh return seat.Pipeline{}, fmt.Errorf("%w: ask %s names pipeline %q, which the runner cannot read: %v", ErrRefused, ev.ID, ev.Pipeline, err) } i := stepOfAsk(p, ev.ID) + // The validate ask is the pipeline's one, naming no mover (H-A-51): its end, naming the mover whose body the holder + // took, is matched by its ask alone and ends every mover's validate step under it (H-A-252). + oneValidate := i >= 0 && p.Steps[i].Step == seat.StepResultStepValidate && p.Steps[i].Seat == wants.Seat + if oneValidate { + i = validateStepOfAsk(p, ev.ID, ev.Module) + } var m seat.Mover if i >= 0 { m, _ = MoverOf(p, p.Steps[i].Module) @@ -248,7 +254,7 @@ func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, sh return p, fmt.Errorf("%w: pipeline %s asked no step under ask %s", ErrRefused, ev.Pipeline, ev.ID) case p.Steps[i].Seat != bs.Seat: return p, fmt.Errorf("%w: ask %s was asked of %s, not %s", ErrRefused, ev.ID, p.Steps[i].Seat, bs.Seat) - case ev.Commit != m.Commit || ev.Fingerprint != m.Fingerprint || ev.Module != m.Module || ev.Step != p.Steps[i].Step: + case ev.Step != p.Steps[i].Step || (!oneValidate && (ev.Commit != m.Commit || ev.Fingerprint != m.Fingerprint || ev.Module != m.Module)): return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s of %s at %s (%s)", ErrRefused, ev.ID, ev.Step, ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Steps[i].Module, m.Commit, m.Fingerprint) case !open(p.Steps[i]): @@ -587,6 +593,32 @@ func endOutcome(o string) bool { return false } +// validateStepOfAsk is the validate step the end of the shared validate ask id ends first (H-A-252): the open one of the mover +// the end names, else any open one under the ask (sameValidate ends the rest), else the first under it. +func validateStepOfAsk(p seat.Pipeline, id, module string) int { + first, anyOpen := -1, -1 + for i, s := range p.Steps { + if s.Step != seat.StepResultStepValidate || s.Ask == "" || (s.Ask != id && AgainID(s.Ask) != id) { + continue + } + if first < 0 { + first = i + } + if open(s) { + if s.Module == module { + return i + } + if anyOpen < 0 { + anyOpen = i + } + } + } + if anyOpen >= 0 { + return anyOpen + } + return first +} + func stepOfAsk(p seat.Pipeline, id string) int { for i, s := range p.Steps { // An ask put again is put under AgainID of the step's ask (H-A-243 (3)): its holder's answer under either is the step's. From 1f878eec580aceda0275c03926cf7d197d0ff029 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 11:55:44 +0100 Subject: [PATCH 6/8] A validate rerun moves every mover's validate and reopens its compositions (H-A-236) The rerun put only the named mover's validate under the new ask; the others stayed failed under the old one, so the pipeline failed again and a second rerun made a second validate ask, breaking the pipeline's one validate ask. The compositions its failed validate ended not asked now wait again, asked once validate passes as at the placing. Builds: design 47 H H-A-51, H-A-236 --- .../internal/pipeline/verbs.go | 28 +++++++++++++ .../internal/pipeline/verbs_test.go | 41 +++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 7568e156..375454ae 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -3,6 +3,7 @@ package pipeline import ( "errors" "fmt" + "slices" "strings" "time" @@ -296,11 +297,15 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error return -1, err } } + oldAsk := s.Ask *s = seat.StepResult{Step: s.Step, Module: s.Module, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint, Seat: s.Seat, Outcome: seat.StepResultOutcomeWaiting, By: caller, Rerun: true} if ask != nil { s.Outcome, s.Started, s.Ask = seat.StepResultOutcomeRunning, &now, ask.ID } + if ask != nil && s.Step == seat.StepResultStepValidate && s.Seat == wants.Seat { + rerunValidate(p, i, oldAsk, caller, now) + } // The pipeline runs again; an approval never asked waits again, to be asked when the step passes. if p.Outcome != seat.PipelineOutcomeRunning { p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil @@ -479,6 +484,29 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &res}, nil } +// rerunValidate puts, with the rerun of validate step i, every other mover's validate step under the same ask back to running +// under its one new id, a mover that left the pipeline skipped, so the pipeline keeps one validate ask (H-A-51); and each +// composition its validate held ended not asked waits again, asked once validate passed as at the placing (§745). +func rerunValidate(p *seat.Pipeline, i int, oldAsk, caller string, now time.Time) { + id := p.Steps[i].Ask + for j := range p.Steps { + st := &p.Steps[j] + if slices.Contains(p.Superseded, st.Module) { + continue + } + switch { + case j != i && st.Step == seat.StepResultStepValidate && st.Seat == wants.Seat && oldAsk != "" && + (st.Ask == oldAsk || st.Ask == AgainID(oldAsk)) && st.Outcome != seat.StepResultOutcomeSuperseded && st.Outcome != seat.StepResultOutcomeCancelled: + *st = seat.StepResult{Step: st.Step, Module: st.Module, Artifact: st.Artifact, Status: st.Status, Fingerprint: st.Fingerprint, Seat: st.Seat, + Outcome: seat.StepResultOutcomeRunning, Started: &now, Ask: id, By: caller, Rerun: true} + case st.Step == seat.StepResultStepComposition && st.Seat == wants.Seat && st.Outcome == seat.StepResultOutcomeNotAsked && + strings.HasPrefix(st.Why, "validate ended "): + *st = seat.StepResult{Step: st.Step, Module: st.Module, Artifact: st.Artifact, Status: st.Status, Fingerprint: st.Fingerprint, Seat: st.Seat, + Outcome: seat.StepResultOutcomeWaiting, Why: "asked once validate passed"} + } + } +} + // passedReader says whether reader's step of run passed, or a person accepted it (§30): a step that moves with no new run. func (v Verbs) passedReader(reader RunReader, run string) bool { p, _, err := v.Hearer.Store.Get(reader.Pipeline) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go index ed9a67c3..6b4fbbac 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go @@ -382,3 +382,44 @@ func TestStartAndAskAgain(t *testing.T) { t.Error("an approval running was asked again") } } + +// Review of H-A-236 (H-A-51): a rerun of validate puts every mover's validate step under the old ask back to running under +// the one new id, a mover that left skipped, and the compositions its failed validate ended not asked wait again; the new +// ask's end ends both validates and asks both compositions. +func TestAValidateRerunMovesEveryMoversValidateAndReopensItsCompositions(t *testing.T) { + v, store, _, asks, _, _ := verbsFixture() + p, rev, _ := store.Get(pkey) + p.Outcome = seat.PipelineOutcomeFailed + p.Movers = append(p.Movers, seat.Mover{Module: "lib", Commit: "c2", Fingerprint: "src2:bb"}, seat.Mover{Module: "gone", Commit: "c3", Fingerprint: "src2:cc"}) + p.Superseded = []string{"gone"} + notAsked := "validate ended failed: nothing is composed" + p.Steps = nil + for _, m := range []string{"app", "lib", "gone"} { + p.Steps = append(p.Steps, + seat.StepResult{Module: m, Step: seat.StepResultStepValidate, Status: seat.StepResultStatusRequired, Seat: wants.Seat, Ask: "old", Outcome: seat.StepResultOutcomeFailed}, + seat.StepResult{Module: m, Step: seat.StepResultStepComposition, Status: seat.StepResultStatusRequired, Seat: wants.Seat, Outcome: seat.StepResultOutcomeNotAsked, Why: notAsked}) + } + p.Steps[4].Outcome, p.Steps[5].Outcome = seat.StepResultOutcomeSuperseded, seat.StepResultOutcomeSuperseded + store.Update(pkey, p, rev) + r, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Module: "app", Step: seat.StepResultStepValidate, Why: "again"}) + if err != nil || len(asks.ids) != 1 || r.Result.Ask != asks.ids[0] { + t.Fatalf("the rerun = %+v (%v), asked %v; want one validate ask", r.Result, err, asks.ids) + } + got, _, _ := store.Get(pkey) + for i, want := range []string{"running " + r.Result.Ask, "waiting ", "running " + r.Result.Ask, "waiting ", "superseded old", "superseded "} { + if s := got.Steps[i]; s.Outcome+" "+s.Ask != want { + t.Errorf("%s's %s = %s %q, want %s", s.Module, s.Step, s.Outcome, s.Ask, want) + } + } + bs := StepSeats[len(StepSeats)-1] + ev := StepEnded{ID: r.Result.Ask, Node: "n1", Pipeline: pkey, Module: "lib", Commit: "c2", Fingerprint: "src2:bb", Step: seat.StepResultStepValidate, Outcome: "passed"} + if _, err := v.Hearer.OnStepEnded(bs, bs.Prefix+"n1", ev); err != nil { + t.Fatal(err) + } + got, _, _ = store.Get(pkey) + for _, i := range []int{1, 3} { + if s := got.Steps[i]; s.Outcome != seat.StepResultOutcomeRunning || s.Ask == "" { + t.Errorf("%s's composition = %s %q, want asked once validate passed", s.Module, s.Outcome, s.Ask) + } + } +} From 374d060cc0be8a07fd2bc127260d79c61d8f727b Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 11:57:44 +0100 Subject: [PATCH 7/8] Take an ask's bound that expired while the runner was down (H-A-238) The bounds were watched for updates only, so an expiry marker written while the runner was down was never heard and that ask had no bound. At the start each ask key's last entry is read; an expiry marker is taken then. A running heard before the seat-holders view is read is heard again after a delay instead of at once. Builds: design 47 H H-A-238 (5), H-A-243 (1) --- .../cmd/mesh-delivery-pipeline/askbounds.go | 5 ++- .../cmd/mesh-delivery-pipeline/main.go | 2 +- .../queue_integration_test.go | 23 +++++++++++ .../cmd/mesh-delivery-pipeline/steps.go | 38 +++++++++++++++++-- 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go index 94d7323e..a6accd91 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go @@ -112,6 +112,9 @@ func hearWantsProgress(hub handlers, c *pipeline.AskClock, logf func(string, ... }) } +// holdersNotReadDelay is how long a running heard before the seat-holders view is read waits before it is heard again. +const holdersNotReadDelay = 2 * time.Second + // hearHolderStarted puts again the open asks of each silent seat whose holder the node-engine says running (H-A-243 (1)). func hearHolderStarted(hub handlers, started func(module string) (int, error), logf func(string, ...any)) error { return hub.Handle(nodeengine.SubjectRunningAll, func(m jetstream.Msg) { @@ -128,7 +131,7 @@ func hearHolderStarted(hub handlers, started func(module string) (int, error), l n, err := started(ev.Module) switch { case errors.Is(err, pipeline.ErrHoldersNotRead): - _ = m.Nak() // heard again once the view is read + _ = m.NakWithDelay(holdersNotReadDelay) // heard again once the view is read, without spinning until then return case err != nil: logf("the asks of the seats %s holds, put again as it runs on %s: %d, and one is not: %v", ev.Module, ev.Node, n, err) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index f5f8af67..8148e161 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -331,7 +331,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro dropExpired(key) } } - if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil { + if err := watchBounds(ctx, bounds, expired, missedAskBound(ctx, bounds), logger.Printf); err != nil { return err } if err := hearDecided(heard, hearer, cred.Module, logger.Printf); err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go index 84a5e92a..e9c797a1 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go @@ -403,7 +403,30 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { if build.Ask == "" || build.Started == nil { t.Fatalf("main.10's build = %+v, want it asked", build) } + // Review of H-A-238 (5): the bound expires while the runner is down; at its next start the expiry marker it left is + // read and taken at once, never a fresh bound's full length later. + boundsKV, err := js.KeyValue(ctx, module+"_bounds") + if err != nil { + t.Fatal(err) + } + waitFor(t, ctx, logs, func() bool { + _, err := boundsKV.Get(ctx, pipeline.AskBoundKey(builds.Seat, build.Ask)) + return err == nil + }) + stopRunner() + waitFor(t, ctx, logs, func() bool { + _, err := boundsKV.Get(ctx, pipeline.AskBoundKey(builds.Seat, build.Ask)) + return errors.Is(err, jetstream.ErrKeyNotFound) + }) + restarted := time.Now() + stopRunner = start() outcome("main.10", seat.PipelineOutcomeFailed) + if e := stepFor(get("main.10"), "app", seat.StepResultStepBuild).Ended; e == nil || e.Sub(restarted) >= 3*time.Second { + t.Fatalf("main.10's build ended %v, the runner restarted at %v; want its missed bound taken at the start", e, restarted) + } + if !strings.Contains(logs.String(), "expired while the runner was away: taken now") { + t.Fatal("the start did not say the bound it took") + } s := stepFor(get("main.10"), "app", seat.StepResultStepBuild) if s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build.Ask+" had no end" || s.Ended == nil || s.Ended.Sub(*build.Started) < 4*time.Second { t.Fatalf("main.10's build = %+v, want could not run, its ask with no end, at its silent bound of 4s", s) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go index 81b53a6a..4d5b85e7 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -182,13 +182,17 @@ func hearAskTaken(hub handlers, taken func(ask, node string) error, c *pipeline. } // watchBounds hears the end of each drop's and each ask's bound, the bus's marker of the key's expiry (A-222, A-77; H-A-235, -// H-A-238 (5)): no timer in the process. -func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error { - w, err := bounds.WatchAll(ctx, jetstream.UpdatesOnly()) +// H-A-238 (5)): no timer in the process. An ask's bound that expired while the runner was down left its marker before the +// watch began: the watch's first entries, each key's last, are read to their end, and each ask's key whose last entry is +// an expiry marker is taken as expired now (missed tells it); a delete marker is a first word, never an expiry. Then every +// expiry is heard as it comes. +func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), missed func(key string) bool, logf func(string, ...any)) error { + w, err := bounds.WatchAll(ctx) if err != nil { return err } go func() { + initial := true for { select { case <-ctx.Done(): @@ -198,9 +202,19 @@ func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(ke if !ok { return } - if e == nil || e.Operation() != jetstream.KeyValuePurge { + if e == nil { + initial = false // the last entries are read: what follows is new continue } + if e.Operation() != jetstream.KeyValuePurge { + continue + } + if initial { + if _, _, _, isAsk := pipeline.ParseAskBound(e.Key()); !isAsk || !missed(e.Key()) { + continue + } + logf("the bound %s expired while the runner was away: taken now", e.Key()) + } expired(e.Key()) } } @@ -208,6 +222,22 @@ func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(ke return nil } +// missedAskBound says whether an ask's bound found expired at the start is still to be taken: a first bound whose ask was +// asked again already (its second bound is held) is not, so a start never asks it once more again. +func missedAskBound(ctx context.Context, bounds jetstream.KeyValue) func(key string) bool { + return func(key string) bool { + onSeat, ask, again, ok := pipeline.ParseAskBound(key) + if !ok { + return false + } + if again { + return true + } + e, err := bounds.Get(ctx, pipeline.AskAgainBoundKey(onSeat, pipeline.AgainID(ask))) + return err != nil || e.Operation() != jetstream.KeyValuePut + } +} + // keptReader reads the kept results of the packages named (§25, §416): the entries of the state kept whose package is a // name or lies beneath it, in one pass over the bucket. func keptReader(ctx context.Context, kept jetstream.KeyValue) func([]string) (map[string][]nodebuildgo.KeptResult, error) { From 6b6e4c81f0dcd394a0ce02131999e3103db16628 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 12:00:22 +0100 Subject: [PATCH 8/8] Judge a missed first bound by the first read's own entries (H-A-238) The start read the second bound with a get, which says not found both for a key never armed and for one the holder's ask-taken deleted, so every later start asked an answered step once more. The first read keeps each key's last entry; a first bound's expiry is taken only when its second bound left no entry at all, and with both passed only the second is taken, ending the step without asking it again. Builds: design 47 H H-A-238 (5) --- .../cmd/mesh-delivery-pipeline/drops_test.go | 25 ++++++++ .../cmd/mesh-delivery-pipeline/main.go | 2 +- .../cmd/mesh-delivery-pipeline/steps.go | 57 +++++++++++-------- 3 files changed, 58 insertions(+), 26 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops_test.go index d10bfce0..d37e6f7e 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops_test.go @@ -2,9 +2,12 @@ package main import ( "context" + "slices" "testing" "time" + "github.com/nats-io/nats.go/jetstream" + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" @@ -50,3 +53,25 @@ func TestADropBoundsExpirySaysCallUnanswered(t *testing.T) { t.Errorf("an answered drop: %v, cleared %v, unbound %v", err, w.cleared, unbound) } } + +// H-A-238 (5), review: at a start, a first bound's expiry is taken only when the first read holds no entry for its second +// bound (put again and taken: a delete marker; put again: a live put; both passed: an expiry, the second taken alone); a +// second bound's expiry is taken; a delete marker or a drop's expiry is none. +func TestMissedBoundsAtTheStart(t *testing.T) { + const s = "node-build-go" + key := func(a string) string { return pipeline.AskBoundKey(s, a) } + again := func(a string) string { return pipeline.AskAgainBoundKey(s, pipeline.AgainID(a)) } + last := map[string]jetstream.KeyValueOp{ + key("lostaaaa000001"): jetstream.KeyValuePurge, // passed while down, never asked again: taken + key("takenaaa000002"): jetstream.KeyValuePurge, again("takenaaa000002"): jetstream.KeyValueDelete, // asked again and taken + key("askedaaa000003"): jetstream.KeyValuePurge, again("askedaaa000003"): jetstream.KeyValuePut, // asked again, bound live + key("bothaaaa000004"): jetstream.KeyValuePurge, again("bothaaaa000004"): jetstream.KeyValuePurge, // both passed + key("wordaaaa000005"): jetstream.KeyValueDelete, // a first word + pipeline.DropBoundKey(s, "dropaaaa000006"): jetstream.KeyValuePurge, + } + want := []string{key("lostaaaa000001"), again("bothaaaa000004")} + slices.Sort(want) + if got := missedBounds(last); !slices.Equal(got, want) { + t.Errorf("missedBounds = %v, want %v", got, want) + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 8148e161..f5f8af67 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -331,7 +331,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro dropExpired(key) } } - if err := watchBounds(ctx, bounds, expired, missedAskBound(ctx, bounds), logger.Printf); err != nil { + if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil { return err } if err := hearDecided(heard, hearer, cred.Module, logger.Printf); err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go index 4d5b85e7..6ae613ff 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "sort" "strings" "github.com/nats-io/nats.go/jetstream" @@ -183,16 +184,15 @@ func hearAskTaken(hub handlers, taken func(ask, node string) error, c *pipeline. // watchBounds hears the end of each drop's and each ask's bound, the bus's marker of the key's expiry (A-222, A-77; H-A-235, // H-A-238 (5)): no timer in the process. An ask's bound that expired while the runner was down left its marker before the -// watch began: the watch's first entries, each key's last, are read to their end, and each ask's key whose last entry is -// an expiry marker is taken as expired now (missed tells it); a delete marker is a first word, never an expiry. Then every -// expiry is heard as it comes. -func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), missed func(key string) bool, logf func(string, ...any)) error { +// watch began: the watch's first entries, each key's last, are kept to their end, and the ask bounds missedBounds names are +// taken as expired then. Then every expiry is heard as it comes. +func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error { w, err := bounds.WatchAll(ctx) if err != nil { return err } go func() { - initial := true + last := map[string]jetstream.KeyValueOp{} // each key's last entry, until the first read ends for { select { case <-ctx.Done(): @@ -203,39 +203,46 @@ func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(ke return } if e == nil { - initial = false // the last entries are read: what follows is new - continue - } - if e.Operation() != jetstream.KeyValuePurge { - continue - } - if initial { - if _, _, _, isAsk := pipeline.ParseAskBound(e.Key()); !isAsk || !missed(e.Key()) { - continue + for _, key := range missedBounds(last) { + logf("the bound %s expired while the runner was away: taken now", key) + expired(key) } - logf("the bound %s expired while the runner was away: taken now", e.Key()) + last = nil // the first read is over: what follows is new + continue + } + if last != nil { + last[e.Key()] = e.Operation() + continue + } + if e.Operation() == jetstream.KeyValuePurge { + expired(e.Key()) } - expired(e.Key()) } } }() return nil } -// missedAskBound says whether an ask's bound found expired at the start is still to be taken: a first bound whose ask was -// asked again already (its second bound is held) is not, so a start never asks it once more again. -func missedAskBound(ctx context.Context, bounds jetstream.KeyValue) func(key string) bool { - return func(key string) bool { +// missedBounds are the ask bounds the first read found expired that are still to be taken (H-A-238 (5)): a second bound's +// expiry always; a first bound's only when the read holds no entry at all for its second bound's key, since a live second +// bound, its expiry or its delete (the holder's ask-taken) says the ask was asked again already. A delete marker is a first +// word, never an expiry; a drop's bound is not taken here. +func missedBounds(last map[string]jetstream.KeyValueOp) []string { + var out []string + for key, op := range last { + if op != jetstream.KeyValuePurge { + continue + } onSeat, ask, again, ok := pipeline.ParseAskBound(key) if !ok { - return false + continue } - if again { - return true + if _, asked := last[pipeline.AskAgainBoundKey(onSeat, pipeline.AgainID(ask))]; again || !asked { + out = append(out, key) } - e, err := bounds.Get(ctx, pipeline.AskAgainBoundKey(onSeat, pipeline.AgainID(ask))) - return err != nil || e.Operation() != jetstream.KeyValuePut } + sort.Strings(out) + return out } // keptReader reads the kept results of the packages named (§25, §416): the entries of the state kept whose package is a