diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go new file mode 100644 index 00000000..a6accd91 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/askbounds.go @@ -0,0 +1,155 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/nats-io/nats.go/jetstream" + + nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// envAskUnansweredBound names the file the manifest writes the setting ask-unanswered-bound into, `${config:}` +// (design 47 H H-A-238 (1)). +const envAskUnansweredBound = "MESH_RUNNER_ASK_UNANSWERED_BOUND_FILE" + +// envAskSilentBound names the file of the setting ask-silent-bound (H-A-243 (2)). +const envAskSilentBound = "MESH_RUNNER_ASK_SILENT_BOUND_FILE" + +// askUnansweredBound reads the setting ask-unanswered-bound at the start, a Go duration (H-A-235, H-A-238 (1)): its default +// ten minutes when the file is missing or empty, an error naming the file for a content that is no positive duration. +func askUnansweredBound(getenv func(string) string) (time.Duration, error) { + return durationSetting(getenv, envAskUnansweredBound, pipeline.DefaultAskUnansweredBound) +} + +// askSilentBound reads the setting ask-silent-bound at the start, a Go duration, forty-five minutes by default (H-A-243 (2)). +func askSilentBound(getenv func(string) string) (time.Duration, error) { + return durationSetting(getenv, envAskSilentBound, pipeline.DefaultAskSilentBound) +} + +func durationSetting(getenv func(string) string, env string, def time.Duration) (time.Duration, error) { + v, err := settingText(getenv(env)) + if err != nil || v == "" { + return def, err + } + d, err := time.ParseDuration(v) + if err != nil || d <= 0 { + return 0, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(env), v) + } + return d, nil +} + +// askClock is the bound of every ask but a test run's over the state bounds (H-A-238 (5)): a key armed with its own time to +// live before the ask is put, deleted at its holder's first word, its expiry heard by watchBounds; the finding call-unanswered, +// scope call, id the ask, raised at the second bound and cleared when the ask is answered after all (H-A-238 (2)). +func askClock(ctx context.Context, bounds jetstream.KeyValue, bound, silent time.Duration, w callFindings, module string, now func() time.Time) *pipeline.AskClock { + return &pipeline.AskClock{Bound: bound, Silent: silent, + Arm: func(key string, ttl time.Duration) error { + _, err := bounds.Create(ctx, key, []byte(`{"ask":"`+key+`"}`), jetstream.KeyTTL(ttl)) + if errors.Is(err, jetstream.ErrKeyExists) { + return nil // armed already: the same ask put again at a start keeps its first bound + } + return err + }, + // A first word is heard for every ask, most of them unbounded or answered already: a key not there is deleted of + // nothing and writes no marker. + Unbound: func(key string) error { + if _, err := bounds.Get(ctx, key); errors.Is(err, jetstream.ErrKeyNotFound) { + return nil + } else if err != nil { + return err + } + if err := bounds.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err + } + return nil + }, + Raise: func(onSeat, ask string, within time.Duration) error { + if w == nil { + return errors.New("the runner has no finding writer") + } + return w.Write(ctx, condseat.Finding{Scope: mcc.ScopeCall, ID: ask, Kind: mcc.KindCallUnanswered, Severity: condseat.FindingSeverityWarning, + Summary: fmt.Sprintf("the ask %s of the seat %s is unanswered", ask, onSeat), + Headline: "A step's ask is not answered", + Explanation: fmt.Sprintf("The runner put the ask %s on the seat %s and heard no word of its holder about it within %s, so its step ended could not run and its pipeline went on (H-A-235, H-A-243).", ask, onSeat, within), + Source: module + "." + mcc.KindCallUnanswered, Observer: module, At: now().UTC(), Subject: condseat.Subject{Scope: mcc.ScopeCall, ID: ask}}) + }, + Clear: func(_, ask string) error { + if w == nil { + return nil + } + return w.Clear(ctx, mcc.ScopeCall, ask, mcc.KindCallUnanswered, "the ask is answered") + }} +} + +// answered deletes the bounds of the ask on onSeat at its holder's first word, said when it fails: the bound then passes and +// its expiry finds the step answered or asks it once more. +func answered(c *pipeline.AskClock, onSeat, ask string, logf func(string, ...any)) { + if err := c.Answered(onSeat, ask); err != nil { + logf("the bounds of ask %s on %s are not deleted at its holder's word: %v", ask, onSeat, err) + } +} + +// hearWantsProgress takes a step progress of the seat mesh-wants as its holder's first word on the ask it names (H-A-238 (3)). +func hearWantsProgress(hub handlers, c *pipeline.AskClock, logf func(string, ...any)) error { + return hub.Handle(wants.SubjectStepProgressAll, func(m jetstream.Msg) { + var ev wants.StepProgressEvent + if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" { + logf("%s is no step-progress: %v", m.Subject(), err) + _ = m.Term() + return + } + answered(c, wants.Seat, ev.Ask, logf) + _ = m.Ack() + }) +} + +// holdersNotReadDelay is how long a running heard before the seat-holders view is read waits before it is heard again. +const holdersNotReadDelay = 2 * time.Second + +// hearHolderStarted puts again the open asks of each silent seat whose holder the node-engine says running (H-A-243 (1)). +func hearHolderStarted(hub handlers, started func(module string) (int, error), logf func(string, ...any)) error { + return hub.Handle(nodeengine.SubjectRunningAll, func(m jetstream.Msg) { + var ev nodeengine.Running + if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Module == "" { + logf("%s is no running: %v", m.Subject(), err) + _ = m.Term() + return + } + if ev.Stopped { + _ = m.Ack() + return + } + n, err := started(ev.Module) + switch { + case errors.Is(err, pipeline.ErrHoldersNotRead): + _ = m.NakWithDelay(holdersNotReadDelay) // heard again once the view is read, without spinning until then + return + case err != nil: + logf("the asks of the seats %s holds, put again as it runs on %s: %d, and one is not: %v", ev.Module, ev.Node, n, err) + } + _ = m.Ack() + }) +} + +// askBoundExpired takes the expiry of an ask's bound (H-A-235): the first asks once more, the second ends its step. +func askBoundExpired(unanswered func(onSeat, ask string, again bool) error, logf func(string, ...any)) func(key string) bool { + return func(key string) bool { + s, ask, again, ok := pipeline.ParseAskBound(key) + if !ok { + return false + } + if err := unanswered(s, ask, again); err != nil { + logf("the bound of ask %s on %s passed and is not taken: %v", ask, s, err) + } + return true + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops.go index 1915061c..b3a3c76f 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops.go @@ -72,8 +72,8 @@ func dropUnanswered(ctx context.Context, w callFindings, module string, now func } } -// boundExpired takes the expiry of a key of the state bounds: a drop's says the drop unanswered (A-222); another key is none -// of the runner's, an ask having no bound of its own (§535, §647 as A-466y amends them). +// boundExpired takes the expiry of a drop's key of the state bounds: it says the drop unanswered (A-222); an ask's key is +// taken by askBoundExpired (H-A-235). func boundExpired(unanswered func(buildSeat, ask string) error, logf func(string, ...any)) func(key string) { return func(key string) { if s, ask, ok := pipeline.ParseDropBound(key); ok { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops_test.go index d10bfce0..d37e6f7e 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops_test.go @@ -2,9 +2,12 @@ package main import ( "context" + "slices" "testing" "time" + "github.com/nats-io/nats.go/jetstream" + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" @@ -50,3 +53,25 @@ func TestADropBoundsExpirySaysCallUnanswered(t *testing.T) { t.Errorf("an answered drop: %v, cleared %v, unbound %v", err, w.cleared, unbound) } } + +// H-A-238 (5), review: at a start, a first bound's expiry is taken only when the first read holds no entry for its second +// bound (put again and taken: a delete marker; put again: a live put; both passed: an expiry, the second taken alone); a +// second bound's expiry is taken; a delete marker or a drop's expiry is none. +func TestMissedBoundsAtTheStart(t *testing.T) { + const s = "node-build-go" + key := func(a string) string { return pipeline.AskBoundKey(s, a) } + again := func(a string) string { return pipeline.AskAgainBoundKey(s, pipeline.AgainID(a)) } + last := map[string]jetstream.KeyValueOp{ + key("lostaaaa000001"): jetstream.KeyValuePurge, // passed while down, never asked again: taken + key("takenaaa000002"): jetstream.KeyValuePurge, again("takenaaa000002"): jetstream.KeyValueDelete, // asked again and taken + key("askedaaa000003"): jetstream.KeyValuePurge, again("askedaaa000003"): jetstream.KeyValuePut, // asked again, bound live + key("bothaaaa000004"): jetstream.KeyValuePurge, again("bothaaaa000004"): jetstream.KeyValuePurge, // both passed + key("wordaaaa000005"): jetstream.KeyValueDelete, // a first word + pipeline.DropBoundKey(s, "dropaaaa000006"): jetstream.KeyValuePurge, + } + want := []string{key("lostaaaa000001"), again("bothaaaa000004")} + slices.Sort(want) + if got := missedBounds(last); !slices.Equal(got, want) { + t.Errorf("missedBounds = %v, want %v", got, want) + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 1597d3e7..f5f8af67 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -67,8 +67,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - // The bound of each superseded step's drop: a key with its own time to live, its expiry heard (A-222, A-77). - bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound a drop no holder answers") + // The bound of each superseded step's drop and of each ask: a key with its own time to live, its expiry heard (A-222, A-77; + // H-A-238 (5)). + bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound a drop or an ask no holder answers") if err != nil { return err } @@ -160,6 +161,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains), Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing} hearer.AskSteps = starter.AskSteps + // Every ask but a test run's is bounded until its holder's first word (H-A-235, H-A-238), its bounds keys of the state + // bounds. + askBound, err := askUnansweredBound(getenv) + if err != nil { + return err + } + silentBound, err := askSilentBound(getenv) + if err != nil { + return err + } + logger.Printf("asks: an ask unanswered bound %s, asked once more, then its step ends could not run; an ask of a seat that says nothing before its end bound %s", askBound, silentBound) + clock := askClock(ctx, bounds, askBound, silentBound, rv.Findings, cred.Module, time.Now) + starter.Asks, hearer.Asks = clock, clock starter.Supersede = func(newer seat.Pipeline) { hearer.SupersedeRunning(newer) } hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) } hearer.AskPinHeld = func(key, module string, pins map[string]string) { @@ -297,10 +311,26 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } return hearer.RunTaken(ask, node) } - if err := hearAskTaken(heard, taken, logger.Printf); err != nil { + if err := hearAskTaken(heard, taken, clock, logger.Printf); err != nil { return err } - expired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf) + if err := hearWantsProgress(heard, clock, logger.Printf); err != nil { + return err + } + // The seat's write verbs are answered by the process that holds the pipelines (A-107); made before the bounds are heard, + // since an ask's expired bound asks it once more through them (H-A-235). + verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls) + verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill} + if err := hearHolderStarted(heard, verbs.HolderStarted, logger.Printf); err != nil { + return err + } + dropExpired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf) + askExpired := askBoundExpired(verbs.AskUnanswered, logger.Printf) + expired := func(key string) { + if !askExpired(key) { + dropExpired(key) + } + } if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil { return err } @@ -336,9 +366,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The steps' ends are kept seat events, each on its own stream (step-ended of the build seats, mesh-builds and // mesh-wants), never on EVENTS: bound here, or no step of any pipeline is ever heard ending (G-1 §24). logger.Printf("the kept seat events heard: %d streams bound", hub.StartKept(ctx, js, cred)) - // The seat's write verbs are answered here, by the process that holds the pipelines (A-107). - verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls) - verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill} stopVerbs, err := answerVerbs(calls, seats, verbs) if err != nil { return err diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go index 8e9fb2d3..e9c797a1 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go @@ -107,6 +107,17 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { const router = "operator-router" answer("mesh.seat."+seatlist.HolderSeat+".call."+seatlist.HolderVerb+".*", map[string]any{"seats": []any{ map[string]any{"seat": oc.Seat, "holders": []any{map[string]any{"module": router}}}}}) + // Each mesh-builds step ask as its holder hears it, on its plain subscription. + var buildAsksMu sync.Mutex + buildAsks := map[string]int{} + if _, err := nc.Subscribe(builds.AcceptSubjectStep, func(m *nats.Msg) { + buildAsksMu.Lock() + buildAsks[m.Header.Get(nats.MsgIdHdr)]++ + buildAsksMu.Unlock() + }); err != nil { + t.Fatal(err) + } + buildAsked := func(id string) int { buildAsksMu.Lock(); defer buildAsksMu.Unlock(); return buildAsks[id] } records, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: router + "_" + oc.StateAsks, History: 1}) if err != nil { t.Fatal(err) @@ -153,6 +164,8 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { if err := os.WriteFile(waitingCap, []byte("1\n"), 0o600); err != nil { t.Fatal(err) } + askBound := filepath.Join(dir, "ask-unanswered-bound") + silentBound := filepath.Join(dir, "ask-silent-bound") logs := &syncBuffer{} start := func() func() { runCtx, stop := context.WithCancel(ctx) @@ -164,6 +177,10 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { return broker case envPipelinesWaitingCap: return waitingCap + case envAskUnansweredBound: + return askBound + case envAskSilentBound: + return silentBound } return "" }, logs) @@ -359,9 +376,16 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { } // At the runner's next start the waiting main.10 is placed from its own state, saying wait-ended, which clears - // pipelines-stalled; the start begins no pipeline of its own. + // pipelines-stalled; the start begins no pipeline of its own. That start's ask-unanswered-bound is one second, its + // ask-silent-bound four. stopRunner() setMaxMsgSize(t, ctx, js, "KV_"+module+"_"+seat.StateQueue, -1) + if err := os.WriteFile(askBound, []byte("1s\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(silentBound, []byte("4s\n"), 0o600); err != nil { + t.Fatal(err) + } before := pipelineKeys(t, ctx, pipelines) stopRunner = start() outcome("main.10", seat.PipelineOutcomeRunning) @@ -370,6 +394,50 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { if after := pipelineKeys(t, ctx, pipelines); !slices.Equal(before, after) { t.Fatalf("the start wrote pipelines %v, want %v", after, before) } + + // H-A-243: main.10's build ask on mesh-builds, whose holder says nothing before its end, is never asked again for + // ask-unanswered-bound (a build running four of those bounds is not touched); its one bound, + // ask-silent-bound, ends its step could not run "its ask had no end", the pipeline ending failed, its turn freed and + // the finding call-unanswered naming the ask. + build := stepFor(get("main.10"), "app", seat.StepResultStepBuild) + if build.Ask == "" || build.Started == nil { + t.Fatalf("main.10's build = %+v, want it asked", build) + } + // Review of H-A-238 (5): the bound expires while the runner is down; at its next start the expiry marker it left is + // read and taken at once, never a fresh bound's full length later. + boundsKV, err := js.KeyValue(ctx, module+"_bounds") + if err != nil { + t.Fatal(err) + } + waitFor(t, ctx, logs, func() bool { + _, err := boundsKV.Get(ctx, pipeline.AskBoundKey(builds.Seat, build.Ask)) + return err == nil + }) + stopRunner() + waitFor(t, ctx, logs, func() bool { + _, err := boundsKV.Get(ctx, pipeline.AskBoundKey(builds.Seat, build.Ask)) + return errors.Is(err, jetstream.ErrKeyNotFound) + }) + restarted := time.Now() + stopRunner = start() + outcome("main.10", seat.PipelineOutcomeFailed) + if e := stepFor(get("main.10"), "app", seat.StepResultStepBuild).Ended; e == nil || e.Sub(restarted) >= 3*time.Second { + t.Fatalf("main.10's build ended %v, the runner restarted at %v; want its missed bound taken at the start", e, restarted) + } + if !strings.Contains(logs.String(), "expired while the runner was away: taken now") { + t.Fatal("the start did not say the bound it took") + } + s := stepFor(get("main.10"), "app", seat.StepResultStepBuild) + if s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build.Ask+" had no end" || s.Ended == nil || s.Ended.Sub(*build.Started) < 4*time.Second { + t.Fatalf("main.10's build = %+v, want could not run, its ask with no end, at its silent bound of 4s", s) + } + if strings.Contains(logs.String(), ": asked once more as ") || buildAsked(pipeline.AgainID(build.Ask)) != 0 { + t.Fatalf("the silent build's ask was asked again for ask-unanswered-bound (put %d times, again %d)", buildAsked(build.Ask), buildAsked(pipeline.AgainID(build.Ask))) + } + waitFor(t, ctx, logs, func() bool { + e, err := findings.Get(ctx, "call."+build.Ask+".call-unanswered") + return err == nil && strings.Contains(string(e.Value()), build.Ask) && len(running()) == 0 + }) } func stepFor(p seat.Pipeline, module, step string) seat.StepResult { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go index c4aa18c1..6ae613ff 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "sort" "strings" "github.com/nats-io/nats.go/jetstream" @@ -141,6 +142,8 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any)) logf("the step-ended of ask %s is not taken yet, heard again: %v", ev.ID, err) _ = m.Nak() default: + // The step's end is its holder's word on the ask, the first for mesh-builds (H-A-238 (3)). + answered(h.Asks, bs.Seat, ev.ID, logf) _ = m.Ack() } }) @@ -153,8 +156,9 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any)) // hearAskTaken keeps on its step the node whose build seat holder took an ask (§602, §647; A-182), so a drop of it is asked of // that node's holder, and begins the clock of a test run's ask (§725). -func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(string, ...any)) error { +func hearAskTaken(hub handlers, taken func(ask, node string) error, c *pipeline.AskClock, logf func(string, ...any)) error { for _, ts := range pipeline.TakenSeats { + ts := ts err := hub.Handle(ts.Subject, func(m jetstream.Msg) { var ev nodebuildgo.AskTaken if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" { @@ -167,6 +171,8 @@ func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(st _ = m.Nak() return } + // The seat's ask-taken is its holder's first word on the ask (H-A-238 (3)). + answered(c, ts.Seat, ev.Ask, logf) _ = m.Ack() }) if err != nil { @@ -176,14 +182,17 @@ func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(st return nil } -// watchBounds hears the end of each drop's bound, the bus's marker of the key's expiry, and says a drop no holder answered -// unanswered (A-222, A-77): no timer in the process. +// watchBounds hears the end of each drop's and each ask's bound, the bus's marker of the key's expiry (A-222, A-77; H-A-235, +// H-A-238 (5)): no timer in the process. An ask's bound that expired while the runner was down left its marker before the +// watch began: the watch's first entries, each key's last, are kept to their end, and the ask bounds missedBounds names are +// taken as expired then. Then every expiry is heard as it comes. func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error { - w, err := bounds.WatchAll(ctx, jetstream.UpdatesOnly()) + w, err := bounds.WatchAll(ctx) if err != nil { return err } go func() { + last := map[string]jetstream.KeyValueOp{} // each key's last entry, until the first read ends for { select { case <-ctx.Done(): @@ -193,16 +202,49 @@ func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(ke if !ok { return } - if e == nil || e.Operation() != jetstream.KeyValuePurge { + if e == nil { + for _, key := range missedBounds(last) { + logf("the bound %s expired while the runner was away: taken now", key) + expired(key) + } + last = nil // the first read is over: what follows is new continue } - expired(e.Key()) + if last != nil { + last[e.Key()] = e.Operation() + continue + } + if e.Operation() == jetstream.KeyValuePurge { + expired(e.Key()) + } } } }() return nil } +// missedBounds are the ask bounds the first read found expired that are still to be taken (H-A-238 (5)): a second bound's +// expiry always; a first bound's only when the read holds no entry at all for its second bound's key, since a live second +// bound, its expiry or its delete (the holder's ask-taken) says the ask was asked again already. A delete marker is a first +// word, never an expiry; a drop's bound is not taken here. +func missedBounds(last map[string]jetstream.KeyValueOp) []string { + var out []string + for key, op := range last { + if op != jetstream.KeyValuePurge { + continue + } + onSeat, ask, again, ok := pipeline.ParseAskBound(key) + if !ok { + continue + } + if _, asked := last[pipeline.AskAgainBoundKey(onSeat, pipeline.AgainID(ask))]; again || !asked { + out = append(out, key) + } + } + sort.Strings(out) + return out +} + // keptReader reads the kept results of the packages named (§25, §416): the entries of the state kept whose package is a // name or lies beneath it, in one pass over the bucket. func keptReader(ctx context.Context, kept jetstream.KeyValue) func([]string) (map[string][]nodebuildgo.KeptResult, error) { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go index e39fabf9..1ad69414 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go @@ -16,6 +16,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" @@ -99,6 +100,19 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { if _, err := holders.Put(ctx, "node-build-go", []byte(`{"holders":[{"module":"mesh-build-go"}]}`)); err != nil { t.Fatal(err) } + // mesh-builds is held by the builder, which takes its asks on a plain subscription and says nothing before an end. + if _, err := holders.Put(ctx, builds.Seat, []byte(`{"holders":[{"module":"mesh-delivery-builder"}]}`)); err != nil { + t.Fatal(err) + } + var buildAsksMu sync.Mutex + buildAsks := map[string]int{} + if _, err := nc.Subscribe(builds.AcceptSubjectStep, func(m *nats.Msg) { + buildAsksMu.Lock() + buildAsks[m.Header.Get(nats.MsgIdHdr)]++ + buildAsksMu.Unlock() + }); err != nil { + t.Fatal(err) + } // The registrations: app tests its program and tools bundle from one directory, its unit step required and its // integration step optional; web is built by node-build-node. @@ -331,6 +345,17 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { const p3 = "main.1" takeIn("app", "c1", "src2:aa", 3, 1) placed(p3) // started by the take-in alone: the runner starts no pipeline by sweep (design 47 H §747) + // H-A-243 (1): the node-engine's running of the builder puts its open build ask again, under its own id: an ask put while + // it was down is lost; a running of a module holding no silent seat puts nothing again. + firstBuild := step(read(p3), "", seat.StepResultStepBuild).Ask + publish(nodeengine.SubjectRunning("n1", "mesh-build-go"), nodeengine.Running{Module: "mesh-build-go", Node: "n1", At: time.Now().UTC()}) + publish(nodeengine.SubjectRunning("n1", "mesh-delivery-builder"), nodeengine.Running{Module: "mesh-delivery-builder", Node: "n1", At: time.Now().UTC()}) + waitFor(t, ctx, logs, func() bool { + buildAsksMu.Lock() + n := buildAsks[firstBuild] + buildAsksMu.Unlock() + return n == 2 && strings.Count(logs.String(), "the ask "+firstBuild+" of the build step of "+p3+" is put again on mesh-builds") == 1 + }) passBuild(p3, "app", "c1", "src2:aa") unitRun, unit := runOfStep(p3, "app:program", "unit") integRun, integ := runOfStep(p3, "app:program", "integration") @@ -350,7 +375,8 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { if s := step(read(p3), "app:tools", "unit"); s.Outcome != seat.StepResultOutcomeWaitingForABuildNode || s.Ask != "" || s.Shared[0].Ask != unit.Ask { t.Fatalf("the tools' unit step = %+v; want it waiting for a build node, reading the program's run", s) } - // Its wait has no bound (§647 as A-466y amends it): the state bounds holds no key for a queued ask, only drops' keys. + // A test run's ask keeps §725's bounds alone (H-A-238 (4)): the state bounds holds no key naming it, only drops' and other + // asks' keys. boundsKV, err := js.KeyValue(ctx, module+"_bounds") if err != nil { t.Fatal(err) @@ -359,8 +385,9 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { t.Fatal(err) } else { for _, k := range ks { - if !strings.HasPrefix(k, pipeline.DropBoundPrefix) { - t.Errorf("the state bounds holds %s after the asks were put; want no bound of a queued ask", k) + _, ask, _, isAsk := pipeline.ParseAskBound(k) + if (!isAsk && !strings.HasPrefix(k, pipeline.DropBoundPrefix)) || ask == unit.Ask || ask == integ.Ask { + t.Errorf("the state bounds holds %s after the asks were put; want no bound of a test run's ask", k) } } } diff --git a/modules/mesh-delivery-pipeline/go.mod b/modules/mesh-delivery-pipeline/go.mod index 58a8de3f..5ea7ce85 100644 --- a/modules/mesh-delivery-pipeline/go.mod +++ b/modules/mesh-delivery-pipeline/go.mod @@ -10,7 +10,7 @@ require ( require ( git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect + git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/asks.go b/modules/mesh-delivery-pipeline/internal/pipeline/asks.go new file mode 100644 index 00000000..e1d1ee1d --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/asks.go @@ -0,0 +1,349 @@ +package pipeline + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strings" + "time" + + builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" +) + +// DefaultAskUnansweredBound is the setting ask-unanswered-bound's default (design 47 H H-A-235, H-A-238 (1)): ten minutes +// from an ask until its holder's first word. +const DefaultAskUnansweredBound = 10 * time.Minute + +// DefaultAskSilentBound is the setting ask-silent-bound's default (H-A-243 (2)): forty-five minutes from an ask on a seat whose +// holder says no word before the step's end (mesh-builds, mesh-wants) until that end. +const DefaultAskSilentBound = 45 * time.Minute + +// Silent says whether the holder of a seat the runner asks says nothing of an ask before its step's end: every seat but the +// build seats, which say ask-taken (H-A-241, H-A-243). +func Silent(onSeat string) bool { return !BoundSeat(onSeat) } + +// AgainID is the id an ask is put again under (H-A-243 (3)): the first id's time part and six hex digits of the sha256 of the +// first id and ".2", the form the holders accept, so no duplicate window is relied on. +func AgainID(id string) string { + if len(id) <= 6 { + return id + } + sum := sha256.Sum256([]byte(id + ".2")) + return id[:len(id)-6] + hex.EncodeToString(sum[:3]) +} + +// The keys of the state bounds that time an ask the runner put (H-A-238 (5)): `ask..` armed before the ask is put, +// `ask2..` armed when it is asked once more, each with the setting as its own time to live. +const ( + AskBoundPrefix = "ask." + AskAgainBoundPrefix = "ask2." +) + +// AskBoundKey is the key of the first bound of the ask on onSeat; AskAgainBoundKey that of its second. +func AskBoundKey(onSeat, ask string) string { return AskBoundPrefix + onSeat + "." + ask } +func AskAgainBoundKey(onSeat, ask string) string { return AskAgainBoundPrefix + onSeat + "." + ask } + +// ParseAskBound reads a key of the state bounds as an ask's bound: the seat, the ask and whether it is the second bound. +func ParseAskBound(key string) (onSeat, ask string, again, ok bool) { + rest, found := strings.CutPrefix(key, AskAgainBoundPrefix) + again = found + if !found { + if rest, found = strings.CutPrefix(key, AskBoundPrefix); !found { + return "", "", false, false + } + } + onSeat, ask, found = strings.Cut(rest, ".") + return onSeat, ask, again, found && onSeat != "" && ask != "" && !strings.Contains(ask, ".") +} + +// UnansweredWhy is the why of a step whose ask was not answered within its two bounds (H-A-235). +func UnansweredWhy(ask string) string { return "its ask " + ask + " was not answered" } + +// SilentWhy is the why of a step on a silent seat whose ask had no end within ask-silent-bound (H-A-243 (2)). +func SilentWhy(ask string) string { return "its ask " + ask + " had no end" } + +// AskClock bounds every ask the runner puts on a seat but a test run's, which keeps §725's bounds (H-A-235, H-A-238 (4)): +// Arm writes a key of the state bounds with Bound as its own time to live, Unbound deletes one; Raise raises the finding +// call-unanswered of an ask that ended its step, Clear clears it once the ask is answered after all (H-A-238 (2)). +type AskClock struct { + // Bound is ask-unanswered-bound, for a seat that says ask-taken; Silent is ask-silent-bound, for a silent seat. + Bound time.Duration + Silent time.Duration + Arm func(key string, ttl time.Duration) error + Unbound func(key string) error + Raise func(onSeat, ask string, bound time.Duration) error + Clear func(onSeat, ask string) error +} + +// arm arms the first bound of a, before it is put (H-A-238 (5)); a test run's ask, and a nil clock, arm none. +func (c *AskClock) arm(a Asked) error { + if c == nil || c.Arm == nil || a.Run != "" { + return nil + } + if err := c.Arm(AskBoundKey(a.Seat, a.ID), c.boundOf(a.Seat)); err != nil { + return fmt.Errorf("the bound of ask %s is not armed: %w", a.ID, err) + } + return nil +} + +// boundOf is the bound of an ask on onSeat: ask-silent-bound for a silent seat (H-A-243 (2)), else ask-unanswered-bound. +func (c *AskClock) boundOf(onSeat string) time.Duration { + if !Silent(onSeat) { + return c.bound() + } + if c.Silent > 0 { + return c.Silent + } + return DefaultAskSilentBound +} + +func (c *AskClock) bound() time.Duration { + if c.Bound > 0 { + return c.Bound + } + return DefaultAskUnansweredBound +} + +// Answered deletes the bounds of the ask heard on onSeat, its holder's first word: an ask-taken, a step progress or, for a +// silent seat, the step's end (H-A-238 (3), H-A-241). Heard under a first id, the second bound of the ask put again under +// AgainID goes too. +func (c *AskClock) Answered(onSeat, ask string) error { + if c == nil || c.Unbound == nil || ask == "" { + return nil + } + return errors.Join(c.Unbound(AskBoundKey(onSeat, ask)), c.Unbound(AskAgainBoundKey(onSeat, ask)), + c.Unbound(AskAgainBoundKey(onSeat, AgainID(ask)))) +} + +// answeredLate clears the finding of the ask of step s, which ended unanswered or with no end, its holder heard after all +// (H-A-238 (2)). +func (c *AskClock) answeredLate(s seat.StepResult, logf func(string, ...any)) { + if c == nil || c.Clear == nil || s.Ask == "" || (s.Why != UnansweredWhy(s.Ask) && s.Why != SilentWhy(s.Ask)) { + return + } + if err := c.Clear(s.Seat, s.Ask); err != nil { + logf("the finding of ask %s, answered after its step ended, is not cleared: %v", s.Ask, err) + } +} + +// AskUnanswered takes the expiry of a bound of the ask on onSeat (H-A-235, H-A-243). On a silent seat its one bound, +// ask-silent-bound, ends the step could not run "its ask had no end", never asking again (that would double a running +// build). On a seat that says ask-taken the first bound puts the ask once more under AgainID, said in the journal, its +// second bound armed under that id; the second ends the step could not run "its ask was not answered". An ended step +// raises the finding call-unanswered, so the pipeline ends and its turn frees. A bound whose ask no running step names any +// longer asks nothing. +func (v Verbs) AskUnanswered(onSeat, ask string, again bool) error { + h, c := v.Hearer, v.Starter.Asks + if c == nil { + return nil + } + key, p, i, err := v.runningAsk(onSeat, ask) + if err != nil || key == "" { + return err + } + s := p.Steps[i] + if Silent(onSeat) || again { + why, within := UnansweredWhy(s.Ask), 2*c.bound() + if Silent(onSeat) { + why, within = SilentWhy(s.Ask), c.boundOf(onSeat) + } + h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s: the step ends could not run (H-A-235, H-A-243)", + s.Ask, s.Step, key, onSeat, within) + if err := h.endUnanswered(key, s.Ask, why); err != nil { + return err + } + if c.Raise != nil { + return c.Raise(onSeat, s.Ask, within) + } + return nil + } + a, err := v.askAgain(p, s, AgainID(s.Ask), h.now()) + if err != nil { + h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s and cannot be put again: %v", s.Ask, s.Step, key, onSeat, c.bound(), err) + } else { + h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s: asked once more as %s (H-A-235, H-A-243)", s.Ask, s.Step, key, onSeat, c.bound(), a.ID) + } + if c.Arm != nil { + if aerr := c.Arm(AskAgainBoundKey(onSeat, AgainID(s.Ask)), c.bound()); aerr != nil { + return fmt.Errorf("the second bound of ask %s is not armed, so it is not asked again: %w", s.Ask, aerr) + } + } + if err != nil { + return nil // its second bound ends the step + } + v.Starter.carry(key, []Asked{*a}) + return askAll(v.Starter.Steps, key, []Asked{*a}, func(commit string) bool { return v.Starter.recovered(key, commit) }, nil) +} + +// HolderStarted puts again every open ask of each silent seat module holds (H-A-243 (1)): its holder takes its asks on a plain +// subscription, so an ask put while it was down is lost; heard on the node-engine's running of module. Each ask is put under +// its own id, a rerun's marks kept, said in the journal; its bound stands as armed. +func (v Verbs) HolderStarted(module string) (int, error) { + st := v.Starter + if st == nil || st.Holders == nil { + return 0, nil + } + var silent []string + for onSeat := range acceptStep { + if !Silent(onSeat) { + continue + } + holders, read := st.Holders(onSeat) + if !read { + return 0, ErrHoldersNotRead + } + if containsString(holders, module) { + silent = append(silent, onSeat) + } + } + if len(silent) == 0 { + return 0, nil + } + pkeys, err := allKeys(v.Hearer.Store) + if err != nil { + return 0, fmt.Errorf("the pipelines are not listed: %w", err) + } + now, asked := v.Hearer.now(), 0 + var first error + for _, key := range pkeys { + p, _, err := v.Hearer.Store.Get(key) + if err != nil || p.Outcome != seat.PipelineOutcomeRunning { + continue + } + seen := map[string]bool{} + var due []Asked + for _, s := range p.Steps { + if !open(s) || s.Ask == "" || seen[s.Ask] || !containsString(silent, s.Seat) { + continue + } + seen[s.Ask] = true + a, err := v.askAgain(p, s, s.Ask, now) + if err != nil { + v.Hearer.logf("the ask %s of the %s step of %s on %s is not put again as %s starts: %v", s.Ask, s.Step, key, s.Seat, module, err) + continue + } + v.Hearer.logf("the ask %s of the %s step of %s is put again on %s: its holder %s started, and takes no ask put while it was down (H-A-243)", + s.Ask, s.Step, key, s.Seat, module) + due = append(due, *a) + } + if len(due) == 0 { + continue + } + st.carry(key, due) + if err := askAll(st.Steps, key, due, func(commit string) bool { return st.recovered(key, commit) }, nil); err != nil && first == nil { + first = err + } + asked += len(due) + } + return asked, first +} + +// ErrHoldersNotRead is HolderStarted's answer before the seat-holders view's first view is read: the running is heard again. +var ErrHoldersNotRead = errors.New("the seat-holders view is not read yet") + +func containsString(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +// runningAsk is the pipeline and the index of its running step asked under ask, or put again under it, of onSeat; an empty +// key when none is. +func (v Verbs) runningAsk(onSeat, ask string) (string, seat.Pipeline, int, error) { + pkeys, err := allKeys(v.Hearer.Store) + if err != nil { + return "", seat.Pipeline{}, -1, fmt.Errorf("the pipelines are not listed: %w", err) + } + for _, key := range pkeys { + p, _, err := v.Hearer.Store.Get(key) + if err != nil { + continue + } + if i := stepOfAsk(p, ask); i >= 0 && open(p.Steps[i]) && p.Steps[i].Seat == onSeat { + return key, p, i, nil + } + } + return "", seat.Pipeline{}, -1, nil +} + +// askAgain is the ask of running step s as it was put, under id: a rerun's marks kept. +func (v Verbs) askAgain(p seat.Pipeline, s seat.StepResult, id string, now time.Time) (*Asked, error) { + a, err := v.placedAsk(p, s, id, now) + if err != nil { + return nil, err + } + if s.Rerun { + markRerun(a, s.By) + } + return a, nil +} + +// markRerun marks a's body a rerun by caller. +func markRerun(a *Asked, caller string) { + switch b := a.Ask.(type) { + case nodebuildgo.StepAsk: + if b.Step != nil { + step := *b.Step + step.Rerun, step.By = true, caller + b.Step = &step + } + a.Ask = b + case builds.StepRun: + b.Rerun, b.By = true, caller + a.Ask = b + case wants.StepRun: + b.Rerun, b.By = true, caller + a.Ask = b + } +} + +// endUnanswered ends the running step of pipeline key asked under ask could not run, saying why (H-A-235, H-A-243): every +// other mover's validate reading the ask with it, a composition its validate held ended not asked, the pipeline ended when +// nothing else is open, said and logged as any step's end. +func (h *Hearer) endUnanswered(key, ask, why string) error { + for { + p, rev, err := h.Store.Get(key) + if err != nil { + return nil + } + i := stepOfAsk(p, ask) + if i < 0 || !open(p.Steps[i]) { + return nil + } + now := h.now() + s := &p.Steps[i] + s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeCouldNotRun, why, &now + p.Unlogged = append(p.Unlogged, endEntry(key, *s, now)) + var readers []int + if s.Step == seat.StepResultStepValidate { + readers = sameValidate(&p, i, now) + _, notAsked := releaseCompositions(&p, now) + readers = append(readers, notAsked...) + } + asked, err := askApproval(key, &p, now, h.Asker) + if err != nil { + return err + } + ended := h.endPipeline(&p, now) + if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return fmt.Errorf("pipeline %s cannot be written: %w", key, err) + } + h.Failures.StepEnded(h.Store, p, *s) + for _, j := range append([]int{i}, readers...) { + if err := h.say(seat.EventStepEnded, p, &p.Steps[j]); err != nil { + return fmt.Errorf("pipeline %s's %s is written but not said: %w", key, seat.EventStepEnded, err) + } + } + h.appendUnlogged(key) + return h.said(p, asked, ended) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go new file mode 100644 index 00000000..1395bdff --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/asks_test.go @@ -0,0 +1,197 @@ +package pipeline + +import ( + "slices" + "strings" + "testing" + "time" + + builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" +) + +// H-A-235, H-A-238: every ask but a test run's is bounded from its put until its holder's first word. A bound that passes asks +// once more under the ask's own id, its second bound armed; past the second the step ends could not run "its ask was not +// answered", the finding raised, and the pipeline ends; a first word deletes both bounds, and a late one clears the finding. +func TestAnAskNobodyAnswersIsAskedOnceMoreThenEndsItsStep(t *testing.T) { + v, store, said, asks, _, _ := verbsFixture() + var armed, unbound, raised, cleared []string + logs := []string{} + v.Hearer.Logf = func(f string, a ...any) { logs = append(logs, f) } + c := &AskClock{Bound: 10 * time.Minute, + Arm: func(key string, ttl time.Duration) error { armed = append(armed, key+" "+ttl.String()); return nil }, + Unbound: func(key string) error { unbound = append(unbound, key); return nil }, + Raise: func(s, ask string, _ time.Duration) error { raised = append(raised, s+" "+ask); return nil }, + Clear: func(s, ask string) error { cleared = append(cleared, s+" "+ask); return nil }} + v.Starter.Asks, v.Hearer.Asks = c, c + + // A test run's ask keeps §725's bounds; a nil clock arms none. + if err := c.arm(Asked{Seat: nodebuildgo.Seat, ID: "r1", Run: "run"}); err != nil || len(armed) != 0 { + t.Fatalf("a test run's ask armed %v (%v)", armed, err) + } + if err := (*AskClock)(nil).arm(Asked{Seat: nodebuildgo.Seat, ID: "x"}); err != nil { + t.Fatal(err) + } + + // Every step but app:program's lint has ended; that lint failed, and its rerun's ask is armed before it is put. + p, rev, _ := store.Get(pkey) + for i := range p.Steps { + if open(p.Steps[i]) { + p.Steps[i].Outcome = seat.StepResultOutcomePassed + } + } + p.Steps[stepIndex(p, "lint", "app:program")].Outcome = seat.StepResultOutcomeFailed + store.Update(pkey, p, rev) + r, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Step: "lint", Artifact: "app:program", Why: "retry"}) + if err != nil { + t.Fatal(err) + } + ask := r.Result.Ask + if want := []string{AskBoundKey(nodebuildgo.Seat, ask) + " 10m0s"}; !slices.Equal(armed, want) || len(asks.ids) != 1 { + t.Fatalf("armed %v asking %v, want %v before one ask", armed, asks.ids, want) + } + // The first bound passes: asked once more under the id derived from the first, in the form the holders accept, a rerun + // still, the second bound armed under it and said (H-A-243 (3)). + again := AgainID(ask) + if err := sdkstep.CheckAskID(again); err != nil || again == ask || again[:len(again)-6] != ask[:len(ask)-6] { + t.Fatalf("AgainID(%s) = %s (%v), want the first's time part and other hex digits in the ask id form", ask, again, err) + } + if err := v.AskUnanswered(nodebuildgo.Seat, ask, false); err != nil { + t.Fatal(err) + } + if len(asks.ids) != 2 || asks.ids[1] != again || armed[1] != AskAgainBoundKey(nodebuildgo.Seat, again)+" 10m0s" { + t.Fatalf("asked %v, armed %v, want %s once more and its second bound", asks.ids, armed, again) + } + if b, ok := asks.asks[1].(nodebuildgo.StepAsk); !ok || b.ID != again || b.Step == nil || !b.Step.Rerun || b.Step.By != "operator" { + t.Errorf("the ask put again = %+v", asks.asks[1]) + } + if !slices.ContainsFunc(logs, func(l string) bool { return strings.Contains(l, "asked once more") }) { + t.Errorf("no journal line names the ask asked again: %v", logs) + } + if got, _, _ := store.Get(pkey); got.Steps[stepIndex(got, "lint", "app:program")].Outcome != seat.StepResultOutcomeRunning { + t.Fatal("the step ended at the first bound") + } + // The second bound passes: the step ends could not run, the finding is raised, the pipeline ends failed. + if err := v.AskUnanswered(nodebuildgo.Seat, again, true); err != nil { + t.Fatal(err) + } + got, _, _ := store.Get(pkey) + s := got.Steps[stepIndex(got, "lint", "app:program")] + if s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+ask+" was not answered" || s.Ended == nil { + t.Errorf("the step = %+v", s) + } + if got.Outcome != seat.PipelineOutcomeFailed || len(asks.ids) != 2 { + t.Errorf("the pipeline is %s (%s), asked %d, want failed and nothing more asked", got.Outcome, got.Why, len(asks.ids)) + } + if !slices.Equal(raised, []string{nodebuildgo.Seat + " " + ask}) || !slices.Contains(*said, "pipeline-ended.main.5") { + t.Errorf("raised %v, said %v", raised, *said) + } + // A bound whose step ended asks nothing; a first word deletes both bounds; a late end clears the finding. + if err := v.AskUnanswered(nodebuildgo.Seat, ask, false); err != nil || len(asks.ids) != 2 { + t.Errorf("a bound of an ended step asked %v (%v)", asks.ids, err) + } + if err := c.Answered(nodebuildgo.Seat, ask); err != nil || !slices.Equal(unbound, []string{AskBoundKey(nodebuildgo.Seat, ask), AskAgainBoundKey(nodebuildgo.Seat, ask), AskAgainBoundKey(nodebuildgo.Seat, again)}) { + t.Errorf("a first word deleted %v (%v)", unbound, err) + } + if err := v.Hearer.Taken(again, "n1"); err != nil || !slices.Equal(cleared, []string{nodebuildgo.Seat + " " + ask}) { + t.Errorf("a late ask-taken cleared %v (%v)", cleared, err) + } + for _, k := range []string{AskBoundKey("mesh-wants", "a1"), AskAgainBoundKey("mesh-wants", "a1")} { + if s, a, again, ok := ParseAskBound(k); !ok || s != "mesh-wants" || a != "a1" || again != strings.HasPrefix(k, AskAgainBoundPrefix) { + t.Errorf("ParseAskBound(%s) = %s %s %v %v", k, s, a, again, ok) + } + } + if _, _, _, ok := ParseAskBound(DropBoundKey("node-build-go", "a1")); ok { + t.Error("a drop's bound read as an ask's") + } +} + +// H-A-243: an ask on a seat whose holder says nothing before the step's end (mesh-builds) is never asked again for a bound: +// its one bound, ask-silent-bound, ends the step could not run "its ask had no end", the finding raised. The node-engine's +// running of the seat's holder puts every open ask of the seat again under its own id, a module holding no such seat none. +func TestASilentSeatsAskIsPutAgainWhenItsHolderStartsAndEndsAtItsSilentBound(t *testing.T) { + v, store, _, asks, _, _ := verbsFixture() + var armed, raised []string + c := &AskClock{Bound: time.Minute, Silent: 45 * time.Minute, + Arm: func(key string, ttl time.Duration) error { armed = append(armed, key+" "+ttl.String()); return nil }, + Raise: func(s, ask string, within time.Duration) error { + raised = append(raised, s+" "+ask+" "+within.String()) + return nil + }} + v.Starter.Asks, v.Hearer.Asks = c, c + v.Starter.Holders = func(s string) ([]string, bool) { + return map[string][]string{builds.Seat: {"mesh-delivery-builder"}}[s], true + } + p, rev, _ := store.Get(pkey) + // Every step has ended but app's build, asked of mesh-builds. + b := stepIndex(p, seat.StepResultStepBuild, "") + for i := range p.Steps { + if open(p.Steps[i]) && i != b { + p.Steps[i].Outcome = seat.StepResultOutcomePassed + } + } + if p.Steps[b].Seat != builds.Seat || !open(p.Steps[b]) { + t.Fatalf("the build step = %+v, want running on mesh-builds", p.Steps[b]) + } + store.Update(pkey, p, rev) + build := p.Steps[b].Ask + if err := c.arm(Asked{Seat: builds.Seat, ID: build}); err != nil || !slices.Equal(armed, []string{AskBoundKey(builds.Seat, build) + " 45m0s"}) { + t.Fatalf("armed %v (%v), want the silent bound", armed, err) + } + // A module holding no silent seat puts nothing again; the builder's running puts the build again under its own id. + if n, err := v.HolderStarted("mesh-build-go"); err != nil || n != 0 || len(asks.ids) != 0 { + t.Fatalf("another holder's start put %d (%v)", n, err) + } + if n, err := v.HolderStarted("mesh-delivery-builder"); err != nil || n != 1 || len(asks.ids) != 1 || asks.ids[0] != build { + t.Fatalf("the builder's start put %d asks %v (%v), want %s again", n, asks.ids, err, build) + } + if _, ok := asks.asks[0].(builds.StepRun); !ok || asks.at[0] != builds.AcceptSubjectStep { + t.Errorf("the ask put again = %+v on %s", asks.asks[0], asks.at[0]) + } + // The silent bound passes: no ask again, the step ends could not run, the finding is raised, the pipeline ends. + if err := v.AskUnanswered(builds.Seat, build, false); err != nil { + t.Fatal(err) + } + got, _, _ := store.Get(pkey) + s := got.Steps[b] + if len(asks.ids) != 1 || s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build+" had no end" || got.Outcome != seat.PipelineOutcomeFailed { + t.Errorf("asked %v; the build %+v; the pipeline %s", asks.ids, s, got.Outcome) + } + if !slices.Equal(raised, []string{builds.Seat + " " + build + " 45m0s"}) { + t.Errorf("raised %v", raised) + } +} + +// H-A-252: the pipeline's one validate ask names no mover (H-A-51), and its holder's end names the mover whose body it took: +// it is matched by its ask alone and ends the validate step of every mover under it, whichever mover it names. +func TestAValidateEndNamingOneMoverEndsEveryMoversValidate(t *testing.T) { + h, store, _, _, _, _ := fixture() + p, rev, _ := store.Get(pkey) + p.Movers = []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}, {Module: "lib", Commit: "c2", Fingerprint: "src2:bb"}} + p.Steps = []seat.StepResult{ + {Module: "app", Step: seat.StepResultStepValidate, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "v1", Seat: wants.Seat}, + {Module: "lib", Step: seat.StepResultStepValidate, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "v1", Seat: wants.Seat}} + store.Update(pkey, p, rev) + var bs StepSeat + for _, x := range StepSeats { + if x.Seat == wants.Seat { + bs = x + } + } + ev := StepEnded{ID: "v1", Node: "n1", Pipeline: pkey, Module: "lib", Commit: "c2", Fingerprint: "src2:bb", Step: seat.StepResultStepValidate, Outcome: "passed"} + if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", ev); err != nil { + t.Fatal(err) + } + got, _, _ := store.Get(pkey) + for _, s := range got.Steps { + if s.Outcome != seat.StepResultOutcomePassed { + t.Errorf("%s's validate = %s, want passed by the one end", s.Module, s.Outcome) + } + } + if got.Outcome != seat.PipelineOutcomePassed { + t.Errorf("the pipeline = %s (%s), want passed", got.Outcome, got.Why) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go b/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go index f3847b25..c95cba8b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go @@ -11,8 +11,8 @@ import ( ) // DropBound is how long a superseded step's drop may go unanswered before it is said (design 47 G A-222, A-77): five minutes, -// a key of the state `bounds` with its own time to live. An ask waiting for its holder has no such bound (§535, §647 as -// A-466y amends them): it is timed by §86's build-capacity finding and, for a test run, by run-untaken-bound (§725). +// a key of the state `bounds` with its own time to live. An ask waiting for its holder is bounded by ask-unanswered-bound +// (H-A-235, H-A-238; AskClock), a test run's by run-untaken-bound (§725). const DropBound = 5 * time.Minute // TakenSeat is a build seat whose holder says `ask-taken` when it takes an ask (design 47 G §602): where the runner hears it. diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index a7dbf327..ef891deb 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -387,17 +387,23 @@ func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Ar Module: m.Module, Pipeline: p.Pipeline, Revision: p.Revision, Source: p.Source, Step: step}} } -// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its -// holder with no bound of the runner's own (§535, §647 as A-466y amends them). -func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit string) bool) error { +// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. arm, when not nil, arms +// each ask's bound before it is put (H-A-235, H-A-238 (5)); an ask whose bound is not armed is not put. +func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit string) bool, arm func(Asked) error) error { var first error for _, a := range asks { accept, ok := acceptStep[a.Seat] if to, named := acceptRecovery[a.Seat]; named && recovery != nil && recovery(a.Commit) { accept = to } - err := fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat) - if ok { + var err error + switch { + case !ok: + err = fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat) + case arm != nil: + err = arm(a) + } + if err == nil { err = asker.AskStep(accept, a.ID, a.Ask) } if err != nil && first == nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go index 54249bca..23b84166 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go @@ -176,6 +176,52 @@ func TestThePipelinesWaitTheirTurn(t *testing.T) { } } +// H-A-244 (1), H-A-223: a mover a newer revision moves leaves an older waiting pipeline also when the newer one is placed at +// once, not only when it waits: the waiting pipeline keeps the movers still its own, and with none left ends superseded, +// never placed. +func TestANewerPlacedPipelineSupersedesAnOlderWaitingOne(t *testing.T) { + s, mem, q, said, _, _, view, _ := queueFixture() + take := takeIn(t, s, view) + take(4, "app") + take(5, "lib", "tool") + if p, _, _ := mem.Get("main.5"); p.Outcome != seat.PipelineOutcomeWaiting { + t.Fatalf("main.5 = %s, want waiting", p.Outcome) + } + s.Queue.AtOnce = 3 + take(6, "lib") + if p, _, _ := mem.Get("main.6"); p.Outcome != seat.PipelineOutcomeRunning { + t.Fatalf("main.6 = %s, want placed at once", p.Outcome) + } + p5, _, _ := mem.Get("main.5") + if p5.Outcome != seat.PipelineOutcomeWaiting || len(p5.Movers) != 1 || p5.Movers[0].Module != "tool" || !slices.Equal(p5.Superseded, []string{"lib"}) { + t.Fatalf("main.5 = %s moving %+v, superseded %v; want waiting with tool alone, lib superseded", p5.Outcome, p5.Movers, p5.Superseded) + } + // At its turn it is placed with tool alone. + p4, rev, _ := mem.Get("main.4") + p4.Outcome = seat.PipelineOutcomePassed + mem.Update("main.4", p4, rev) + if _, err := s.Turn("main.4", view); err != nil { + t.Fatal(err) + } + if p, _, _ := mem.Get("main.5"); p.Outcome != seat.PipelineOutcomeRunning || len(stepsOf(p, "lib")) != 0 || len(stepsOf(p, "tool")) == 0 { + t.Fatalf("main.5 at its turn = %s with steps %+v, want running with tool's steps alone", p.Outcome, p.Steps) + } + + // A waiting pipeline whose one mover a newer placed pipeline moves ends superseded, never placed. + s, mem, q, said, _, _, view, _ = queueFixture() + take = takeIn(t, s, view) + take(4, "app") + take(5, "tool") + s.Queue.AtOnce = 3 + take(7, "tool") + if p, _, _ := mem.Get("main.5"); p.Outcome != seat.PipelineOutcomeSuperseded || len(p.Steps) != 0 || p.Ended == nil { + t.Errorf("main.5 = %s with %d steps, want superseded and never placed", p.Outcome, len(p.Steps)) + } + if _, ok := q["waiting.00000000000000000001"]; ok || !slices.Contains(*said, "pipeline-ended.main.5") { + t.Errorf("the queue %v, said %v; want main.5's waiting key gone and its pipeline-ended said", q, *said) + } +} + // R13/AC1 (design 47 H §748, ADR 0682 §7): a newer waiting pipeline of the same branch takes the movers it also moves out of // an older waiting one, which ends superseded, saying wait-ended and pipeline-ended, once no mover is left; a running // pipeline moving the same module runs to its end. diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 68dd3f35..17168081 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -183,6 +183,8 @@ type Starter struct { // Queue bounds the pipelines asked at once to pipelines-at-once, the others waiting in order (design 47 H §748); nil // places every start at once. Queue *Queue + // Asks bounds every ask but a test run's until its holder's first word (H-A-235, H-A-238); nil bounds none. + Asks *AskClock // RunTurn places a run on request whose turn in the queue came, from its candidate's newest revision (A-222); nil places // it at the runner's next start. RunTurn func(UnplacedRun) @@ -252,7 +254,7 @@ func (s *Starter) askSteps(key string, asks []Asked) error { var err error if len(asks) > 0 { s.carry(key, asks) - err = askAll(s.Steps, key, asks, func(commit string) bool { return s.recovered(key, commit) }) + err = askAll(s.Steps, key, asks, func(commit string) bool { return s.recovered(key, commit) }, s.Asks.arm) } s.sayStarted(key, placed) return err @@ -545,11 +547,12 @@ func (s *Starter) Start(b Begin) (bool, error) { if s.Supersede != nil { s.Supersede(p) } + // ... and every older waiting one, whether this one waits or is placed at once (H-A-244 (1)). + if _, err := s.supersedeWaiting(p); err != nil { + s.logf("the older waiting pipelines of %s are not superseded by %s: %v", p.Branch, key, err) + } if !place { s.sayQueueWait(seat.EventWaitHeld, p, now, "") - if _, err := s.supersedeWaiting(p); err != nil { - s.logf("the older waiting pipelines of %s are not superseded by %s: %v", p.Branch, key, err) - } s.Queue.overCap(key, s.logf) return true, nil } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 78a4d3f8..5b1b8edf 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -176,6 +176,9 @@ type Hearer struct { // Unplaced says why a mover of a pipeline has no step placed, for the why of its end (H-A-171; Starter.Unplaced): it // waits on its registration or on a publish; nil says only that it has no step placed. Unplaced func(p seat.Pipeline, module string) string + // Asks clears the finding of an ask that ended its step unanswered once it is answered after all (H-A-238 (2)); nil + // clears none. + Asks *AskClock // Rejoin keys a pipeline running again once its approval is answered and it still runs (H-A-106 (b); Starter.Rejoin); // nil keys none. Rejoin func(key string) @@ -236,6 +239,12 @@ func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, sh return seat.Pipeline{}, fmt.Errorf("%w: ask %s names pipeline %q, which the runner cannot read: %v", ErrRefused, ev.ID, ev.Pipeline, err) } i := stepOfAsk(p, ev.ID) + // The validate ask is the pipeline's one, naming no mover (H-A-51): its end, naming the mover whose body the holder + // took, is matched by its ask alone and ends every mover's validate step under it (H-A-252). + oneValidate := i >= 0 && p.Steps[i].Step == seat.StepResultStepValidate && p.Steps[i].Seat == wants.Seat + if oneValidate { + i = validateStepOfAsk(p, ev.ID, ev.Module) + } var m seat.Mover if i >= 0 { m, _ = MoverOf(p, p.Steps[i].Module) @@ -245,10 +254,11 @@ func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, sh return p, fmt.Errorf("%w: pipeline %s asked no step under ask %s", ErrRefused, ev.Pipeline, ev.ID) case p.Steps[i].Seat != bs.Seat: return p, fmt.Errorf("%w: ask %s was asked of %s, not %s", ErrRefused, ev.ID, p.Steps[i].Seat, bs.Seat) - case ev.Commit != m.Commit || ev.Fingerprint != m.Fingerprint || ev.Module != m.Module || ev.Step != p.Steps[i].Step: + case ev.Step != p.Steps[i].Step || (!oneValidate && (ev.Commit != m.Commit || ev.Fingerprint != m.Fingerprint || ev.Module != m.Module)): return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s of %s at %s (%s)", ErrRefused, ev.ID, ev.Step, ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Steps[i].Module, m.Commit, m.Fingerprint) case !open(p.Steps[i]): + h.Asks.answeredLate(p.Steps[i], h.logf) if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil { if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID { // The asking step's end was written and its run's was not (a write failed): the run ends now, heard again. @@ -583,9 +593,36 @@ func endOutcome(o string) bool { return false } +// validateStepOfAsk is the validate step the end of the shared validate ask id ends first (H-A-252): the open one of the mover +// the end names, else any open one under the ask (sameValidate ends the rest), else the first under it. +func validateStepOfAsk(p seat.Pipeline, id, module string) int { + first, anyOpen := -1, -1 + for i, s := range p.Steps { + if s.Step != seat.StepResultStepValidate || s.Ask == "" || (s.Ask != id && AgainID(s.Ask) != id) { + continue + } + if first < 0 { + first = i + } + if open(s) { + if s.Module == module { + return i + } + if anyOpen < 0 { + anyOpen = i + } + } + } + if anyOpen >= 0 { + return anyOpen + } + return first +} + func stepOfAsk(p seat.Pipeline, id string) int { for i, s := range p.Steps { - if id != "" && s.Ask == id { + // An ask put again is put under AgainID of the step's ask (H-A-243 (3)): its holder's answer under either is the step's. + if id != "" && s.Ask != "" && (s.Ask == id || AgainID(s.Ask) == id) { return i } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go index 104c792b..a040e507 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go @@ -363,7 +363,7 @@ func TestARecoverysStepsAreAskedOnTheStreamDelivery(t *testing.T) { }{{true, "mesh.seat.node-build-go.accept.delivery.mesh-pipeline-runner," + wants.AcceptSubjectStep}, {false, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner," + wants.AcceptSubjectStep}} { var to []string asker := askerFunc(func(accept, _ string, _ any) error { to = append(to, accept); return nil }) - if err := askAll(asker, "k", asks, func(string) bool { return c.recovery }); err != nil || strings.Join(to, ",") != c.want { + if err := askAll(asker, "k", asks, func(string) bool { return c.recovery }, nil); err != nil || strings.Join(to, ",") != c.want { t.Errorf("recovery %v: asked on %v (%v), want %s", c.recovery, to, err, c.want) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go index 6d77cbda..0f2340ad 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go @@ -34,6 +34,9 @@ func (h *Hearer) Taken(ask, node string) error { break } i := stepOfAsk(p, ask) + if i >= 0 && !open(p.Steps[i]) { + h.Asks.answeredLate(p.Steps[i], h.logf) + } if i < 0 || !open(p.Steps[i]) || p.Steps[i].Node == node { break } @@ -163,10 +166,11 @@ func (h *Hearer) supersedeRunningOne(key string, newer seat.Pipeline) bool { // supersedeWaiting takes, in each older waiting pipeline of newer's branch, the movers newer also moves out of it (design 47 H // §748, ADR 0682 §7): a waiting pipeline left with no mover ends superseded, its waiting key deleted, saying wait-ended and its -// pipeline-ended. newer is a pipeline just written waiting; a running pipeline's movers leave it by SupersedeRunning -// (H-A-223). It answers the pipelines it ended. +// pipeline-ended. newer is a pipeline just written, waiting or placed: a mover a newer revision moves leaves every pipeline not +// ended, a waiting one as a running one (H-A-223, H-A-244 (1)), so a waiting pipeline is placed with only the movers still its +// own; a running pipeline's movers leave it by SupersedeRunning. It answers the pipelines it ended. func (s *Starter) supersedeWaiting(newer seat.Pipeline) ([]string, error) { - if s.Queue == nil || newer.Outcome != seat.PipelineOutcomeWaiting { + if s.Queue == nil || (newer.Outcome != seat.PipelineOutcomeWaiting && newer.Outcome != seat.PipelineOutcomeRunning) { return nil, nil } wkeys, pipelines, err := s.Queue.Waiting() @@ -193,7 +197,7 @@ func (s *Starter) supersedeWaiting(newer seat.Pipeline) ([]string, error) { func (s *Starter) supersedeOne(wkey, key string, newer seat.Pipeline) (bool, error) { for { p, rev, err := s.Store.Get(key) - if err != nil || p.Outcome != seat.PipelineOutcomeWaiting || p.Branch != newer.Branch || p.Seq >= newer.Seq { + if err != nil || p.Outcome != seat.PipelineOutcomeWaiting || p.Branch != newer.Branch || !olderWaiting(p, newer) { return false, nil } left := slices.DeleteFunc(slices.Clone(p.Movers), func(m seat.Mover) bool { return Moves(newer, m.Module) }) @@ -232,6 +236,15 @@ func (s *Starter) supersedeOne(wkey, key string, newer seat.Pipeline) (bool, err } } +// olderWaiting says whether the waiting pipeline p is older than newer: by its sequence in the queue when newer waits too, +// by its catalog revision when newer was placed at once (H-A-244 (1)); a run on request's pipelines are never compared. +func olderWaiting(p, newer seat.Pipeline) bool { + if newer.Outcome == seat.PipelineOutcomeWaiting { + return p.Seq < newer.Seq + } + return p.Run == "" && newer.Run == "" && p.Revision < newer.Revision +} + // supersededWhy writes, into mover module's build steps, that a dependency it builds on was taken out of p by a newer // pipeline, so it builds against the newest published package (design 47 H §748; H-A-53, H-A-107). func supersededWhy(p seat.Pipeline, module string, entries map[string]catalog.CatalogEntry, steps []seat.StepResult) { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index d86cec3b..375454ae 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -3,10 +3,10 @@ package pipeline import ( "errors" "fmt" + "slices" "strings" "time" - builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" @@ -297,11 +297,15 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error return -1, err } } + oldAsk := s.Ask *s = seat.StepResult{Step: s.Step, Module: s.Module, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint, Seat: s.Seat, Outcome: seat.StepResultOutcomeWaiting, By: caller, Rerun: true} if ask != nil { s.Outcome, s.Started, s.Ask = seat.StepResultOutcomeRunning, &now, ask.ID } + if ask != nil && s.Step == seat.StepResultStepValidate && s.Seat == wants.Seat { + rerunValidate(p, i, oldAsk, caller, now) + } // The pipeline runs again; an approval never asked waits again, to be asked when the step passes. if p.Outcome != seat.PipelineOutcomeRunning { p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil @@ -321,7 +325,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error } if ask != nil { v.Starter.carry(in.Pipeline, []Asked{*ask}) - if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }); err != nil { + if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm); err != nil { return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err) } } @@ -468,7 +472,7 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) break } v.Starter.carry(in.Pipeline, []Asked{*ask}) - if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }); err != nil { + if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm); err != nil { return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err) } } @@ -480,6 +484,29 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &res}, nil } +// rerunValidate puts, with the rerun of validate step i, every other mover's validate step under the same ask back to running +// under its one new id, a mover that left the pipeline skipped, so the pipeline keeps one validate ask (H-A-51); and each +// composition its validate held ended not asked waits again, asked once validate passed as at the placing (§745). +func rerunValidate(p *seat.Pipeline, i int, oldAsk, caller string, now time.Time) { + id := p.Steps[i].Ask + for j := range p.Steps { + st := &p.Steps[j] + if slices.Contains(p.Superseded, st.Module) { + continue + } + switch { + case j != i && st.Step == seat.StepResultStepValidate && st.Seat == wants.Seat && oldAsk != "" && + (st.Ask == oldAsk || st.Ask == AgainID(oldAsk)) && st.Outcome != seat.StepResultOutcomeSuperseded && st.Outcome != seat.StepResultOutcomeCancelled: + *st = seat.StepResult{Step: st.Step, Module: st.Module, Artifact: st.Artifact, Status: st.Status, Fingerprint: st.Fingerprint, Seat: st.Seat, + Outcome: seat.StepResultOutcomeRunning, Started: &now, Ask: id, By: caller, Rerun: true} + case st.Step == seat.StepResultStepComposition && st.Seat == wants.Seat && st.Outcome == seat.StepResultOutcomeNotAsked && + strings.HasPrefix(st.Why, "validate ended "): + *st = seat.StepResult{Step: st.Step, Module: st.Module, Artifact: st.Artifact, Status: st.Status, Fingerprint: st.Fingerprint, Seat: st.Seat, + Outcome: seat.StepResultOutcomeWaiting, Why: "asked once validate passed"} + } + } +} + // passedReader says whether reader's step of run passed, or a person accepted it (§30): a step that moves with no new run. func (v Verbs) passedReader(reader RunReader, run string) bool { p, _, err := v.Hearer.Store.Get(reader.Pipeline) @@ -500,8 +527,24 @@ func (v Verbs) passedReader(reader RunReader, run string) bool { } // askOf is the ask to put on the serving seat for step s again: the one Place makes for it, under a new id (an ask put twice -// under one id is kept once), saying it is a rerun and by whom. +// under one id is kept once), saying it is a rerun and by whom. The validate ask is the pipeline's one, naming no mover +// (H-A-51): its rerun asks that one ask again; a composition's rerun builds the mover's composition ask (H-A-236). func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time.Time) (*Asked, error) { + id := AskID(now, p.Pipeline, s.Module, s.Artifact, s.Step) + if s.Seat == wants.Seat && s.Step == seat.StepResultStepValidate { + id = AskID(now, p.Pipeline, "", "", seat.StepResultStepValidate) + } + a, err := v.placedAsk(p, s, id, now) + if err != nil { + return nil, err + } + markRerun(a, caller) + return a, nil +} + +// placedAsk is the ask of step s under id: a validate or composition step of the seat mesh-wants built directly for its +// mover (H-A-236), any other the one Place makes for it. +func (v Verbs) placedAsk(p seat.Pipeline, s seat.StepResult, id string, now time.Time) (*Asked, error) { if v.Starter == nil || v.Starter.Registration == nil || v.Starter.Steps == nil || (v.View == nil && p.Run == "") { return nil, errors.New("the runner cannot ask a step: it holds no registration or no ask") } @@ -509,6 +552,16 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time if !ok { return nil, fmt.Errorf("%s moves no module %q", p.Pipeline, s.Module) } + if s.Seat == wants.Seat { + switch s.Step { + case seat.StepResultStepValidate: + a := wantsAsk(p, m, id, wants.StepRunStepValidate) + return &a, nil + case seat.StepResultStepComposition: + a := wantsAsk(p, m, id, wants.StepRunStepComposition) + return &a, nil + } + } reg, ok := v.Starter.Registration(m.Module) if !ok { return nil, fmt.Errorf("%s has no registration", m.Module) @@ -533,21 +586,9 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time if a.ID != base { continue } - a.ID = AskID(now, p.Pipeline, m.Module, s.Artifact, s.Step) - switch b := a.Ask.(type) { - case nodebuildgo.StepAsk: - b.ID = a.ID - if b.Step != nil { - step := *b.Step - step.Rerun, step.By = true, caller - b.Step = &step - } - a.Ask = b - case builds.StepRun: - b.Rerun, b.By = true, caller - a.Ask = b - case wants.StepRun: - b.Rerun, b.By = true, caller + a.ID = id + if b, ok := a.Ask.(nodebuildgo.StepAsk); ok { + b.ID = id a.Ask = b } return &a, nil diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go index a2c8adae..6b4fbbac 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go @@ -8,6 +8,7 @@ import ( catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" "git.novox.be/novox/mesh-sdk/go/views" ) @@ -252,6 +253,47 @@ func TestRerunOfARunAsksWithItsCandidatesHalf(t *testing.T) { } } +// H-A-236: a rerun of a failed pipeline's validate step asks the pipeline's one validate ask again (H-A-51), naming no +// mover in its id; a rerun of a composition step builds the mover's composition ask directly. Neither is refused "placed no +// ask now". +func TestRerunOfValidateOrCompositionAsksTheSharedAsk(t *testing.T) { + v, store, _, asks, _, _ := verbsFixture() + reg, entries := placeFixture() + reg.Policy.Steps.Validate, reg.Policy.Steps.Composition = "required", "required" + v.Starter.Registration = func(string) (catalog.Registration, bool) { return reg, true } + p, rev, _ := store.Get(pkey) + p.Steps, _ = Place(p, p.Movers[0], reg, entries, time.Unix(1, 0), "the-runner") + for i := range p.Steps { + switch p.Steps[i].Step { + case seat.StepResultStepValidate: + p.Steps[i].Outcome = seat.StepResultOutcomeFailed + case seat.StepResultStepComposition: + p.Steps[i].Outcome = seat.StepResultOutcomeNotAsked + } + } + p.Outcome = seat.PipelineOutcomeFailed + store.Update(pkey, p, rev) + now := v.now() + for _, c := range []struct{ step, module string }{{seat.StepResultStepValidate, ""}, {seat.StepResultStepComposition, "app"}} { + n := len(asks.asks) + r, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Module: "app", Step: c.step, Why: "the holder restarted"}) + want := AskID(now, pkey, c.module, "", c.step) + if err != nil || r.Result == nil || r.Result.Ask != want || r.Result.Outcome != seat.StepResultOutcomeRunning { + t.Fatalf("a rerun of %s = %+v, %v, want running under %s", c.step, r.Result, err, want) + } + if len(asks.asks) != n+1 || asks.ids[n] != want { + t.Fatalf("a rerun of %s asked %v, want one ask under %s", c.step, asks.ids[n:], want) + } + b, ok := asks.asks[n].(wants.StepRun) + if !ok || b.Step != c.step || b.Module != "app" || b.Pipeline != pkey || !b.Rerun || b.By != "operator" || b.Asker != seat.Seat { + t.Errorf("the %s ask = %+v", c.step, asks.asks[n]) + } + p, rev, _ = store.Get(pkey) + p.Steps[stepIndex(p, c.step, "")].Outcome = seat.StepResultOutcomeFailed + store.Update(pkey, p, rev) + } +} + func stepIndex(p seat.Pipeline, step, artifact string) int { for i, s := range p.Steps { if s.Step == step && s.Artifact == artifact { @@ -340,3 +382,44 @@ func TestStartAndAskAgain(t *testing.T) { t.Error("an approval running was asked again") } } + +// Review of H-A-236 (H-A-51): a rerun of validate puts every mover's validate step under the old ask back to running under +// the one new id, a mover that left skipped, and the compositions its failed validate ended not asked wait again; the new +// ask's end ends both validates and asks both compositions. +func TestAValidateRerunMovesEveryMoversValidateAndReopensItsCompositions(t *testing.T) { + v, store, _, asks, _, _ := verbsFixture() + p, rev, _ := store.Get(pkey) + p.Outcome = seat.PipelineOutcomeFailed + p.Movers = append(p.Movers, seat.Mover{Module: "lib", Commit: "c2", Fingerprint: "src2:bb"}, seat.Mover{Module: "gone", Commit: "c3", Fingerprint: "src2:cc"}) + p.Superseded = []string{"gone"} + notAsked := "validate ended failed: nothing is composed" + p.Steps = nil + for _, m := range []string{"app", "lib", "gone"} { + p.Steps = append(p.Steps, + seat.StepResult{Module: m, Step: seat.StepResultStepValidate, Status: seat.StepResultStatusRequired, Seat: wants.Seat, Ask: "old", Outcome: seat.StepResultOutcomeFailed}, + seat.StepResult{Module: m, Step: seat.StepResultStepComposition, Status: seat.StepResultStatusRequired, Seat: wants.Seat, Outcome: seat.StepResultOutcomeNotAsked, Why: notAsked}) + } + p.Steps[4].Outcome, p.Steps[5].Outcome = seat.StepResultOutcomeSuperseded, seat.StepResultOutcomeSuperseded + store.Update(pkey, p, rev) + r, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Module: "app", Step: seat.StepResultStepValidate, Why: "again"}) + if err != nil || len(asks.ids) != 1 || r.Result.Ask != asks.ids[0] { + t.Fatalf("the rerun = %+v (%v), asked %v; want one validate ask", r.Result, err, asks.ids) + } + got, _, _ := store.Get(pkey) + for i, want := range []string{"running " + r.Result.Ask, "waiting ", "running " + r.Result.Ask, "waiting ", "superseded old", "superseded "} { + if s := got.Steps[i]; s.Outcome+" "+s.Ask != want { + t.Errorf("%s's %s = %s %q, want %s", s.Module, s.Step, s.Outcome, s.Ask, want) + } + } + bs := StepSeats[len(StepSeats)-1] + ev := StepEnded{ID: r.Result.Ask, Node: "n1", Pipeline: pkey, Module: "lib", Commit: "c2", Fingerprint: "src2:bb", Step: seat.StepResultStepValidate, Outcome: "passed"} + if _, err := v.Hearer.OnStepEnded(bs, bs.Prefix+"n1", ev); err != nil { + t.Fatal(err) + } + got, _, _ = store.Get(pkey) + for _, i := range []int{1, 3} { + if s := got.Steps[i]; s.Outcome != seat.StepResultOutcomeRunning || s.Ask == "" { + t.Errorf("%s's composition = %s %q, want asked once validate passed", s.Module, s.Outcome, s.Ask) + } + } +} diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index e500b0a8..7de59100 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -84,7 +84,7 @@ func TestManifest(t *testing.T) { "mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended", "operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended", "node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", - "node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-catalog.registration-changed", + "node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-wants.step-progress", "node-engine.running", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-catalog.version-released", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged", // A-186: the seat list its write verbs' callers are checked against, heard as it changes. "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", @@ -199,7 +199,7 @@ func runSettingsOf(raw []byte) bool { if json.Unmarshal(raw, &m) != nil { return false } - want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "widens-read-bound": "string", "revisions-kept": "number", + want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "ask-unanswered-bound": "string", "ask-silent-bound": "string", "widens-read-bound": "string", "revisions-kept": "number", "pipelines-at-once": "number", "pipelines-waiting-cap": "number"} if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) || m.Config["pipelines-at-once"]["minimum"] != float64(1) || m.Config["pipelines-at-once"]["maximum"] != float64(8) || diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index 671a0b97..d25b519d 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -58,6 +58,8 @@ "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", + "mesh-wants.step-progress", + "node-engine.running", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-catalog.version-released", @@ -151,6 +153,18 @@ "description": "how long a test run's ask may wait untaken after the run's start before the run ends could not run, a Go duration such as 24h (design 47 G section 725, A-466s, A-469x).", "trusted": false, "default": "24h" + }, + "ask-unanswered-bound": { + "type": "string", + "description": "how long an ask the runner puts on a seat, but a test run's, may go without its holder's first word (an ask-taken, a step progress or the step's end) before it is asked once more, and as long again before its step ends could not run, a Go duration such as 10m (design 47 H H-A-235, H-A-238).", + "trusted": false, + "default": "10m" + }, + "ask-silent-bound": { + "type": "string", + "description": "how long an ask the runner puts on a seat whose holder says nothing before the step's end (mesh-builds, mesh-wants) may go without that end before its step ends could not run, a Go duration such as 45m (design 47 H H-A-243).", + "trusted": false, + "default": "45m" } }, "state": [ @@ -278,6 +292,22 @@ "mode": "0600", "content": "${config:run-untaken-bound}" }, + { + "id": "ask-unanswered-bound", + "type": "file", + "path": "${dir:state}/ask-unanswered-bound", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:ask-unanswered-bound}" + }, + { + "id": "ask-silent-bound", + "type": "file", + "path": "${dir:state}/ask-silent-bound", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:ask-silent-bound}" + }, { "id": "widens-read-bound", "type": "file", @@ -326,6 +356,8 @@ "MESH_RUNNER_WIDENS_READ_BOUND_FILE": "${dir:state}/widens-read-bound", "MESH_RUNNER_REVISIONS_KEPT_FILE": "${dir:state}/revisions-kept", "MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound", + "MESH_RUNNER_ASK_UNANSWERED_BOUND_FILE": "${dir:state}/ask-unanswered-bound", + "MESH_RUNNER_ASK_SILENT_BOUND_FILE": "${dir:state}/ask-silent-bound", "MESH_RUNNER_PIPELINES_AT_ONCE_FILE": "${dir:state}/pipelines-at-once", "MESH_RUNNER_PIPELINES_WAITING_CAP_FILE": "${dir:state}/pipelines-waiting-cap" },