diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go index eb5f47fb..3a4f3e7b 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go @@ -46,7 +46,7 @@ func as[T any](args map[string]any) (T, error) { return in, nil } -// Tools are the runner's read verbs over its states pipelines and kept. +// Tools are the runner's read verbs over its states pipelines and kept; test-runs is the process's (A-470i). func Tools(r pipeline.Reads) []stdio.Tool { runs := map[string]func(map[string]any) (any, error){ seat.VerbShow: func(a map[string]any) (any, error) { @@ -152,6 +152,7 @@ func main() { tools = append(tools, UnloggedTools(stdio.State(seat.StatePipelines), stdio.Log(seat.StateStepLog), meshhandactsclient.Recorder{Ask: stdio.Ask}, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })...) tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask)) + tools = append(tools, TestRunsTools(stdio.Ask)...) tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now)) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/testruns.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/testruns.go new file mode 100644 index 00000000..39800750 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/testruns.go @@ -0,0 +1,43 @@ +package main + +import ( + "encoding/json" + "errors" + "strings" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// TestRunsTools are runner_test_runs and runner_test_runs_drop (design 47 G §728; A-469f, A-469u): each calls its seat verb +// over the bus, both answered by the runner's process, which holds test-runs-cap-mib (A-469g, A-470i). +func TestRunsTools(ask func(key string, body any) (json.RawMessage, error)) []stdio.Tool { + return []stdio.Tool{ + {Name: Prefix + "_test_runs", + Description: "The pipeline runner's test runs, one per candidate, level and tested packages (the state test-runs): each with its packages read from its output object, its ask, node, readers, outcome and counts, and the state's fill against its cap setting test-runs-cap-mib; filtered by a candidate, a level, a digest or a module, or none.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "candidate": map[string]any{"type": "string"}, "level": map[string]any{"type": "string"}, + "digest": map[string]any{"type": "string"}, "module": map[string]any{"type": "string"}}}, + Run: func(args map[string]any) (any, error) { + in, err := as[seat.TestRunsInput](args) + if err != nil { + return nil, err + } + return ask("seat:"+seat.Seat+"."+seat.VerbTestRuns, in) + }}, + {Name: Prefix + "_test_runs_drop", + Description: "Drop one test run by the seat verb test-runs-drop, a hand act recorded first: its ask is dropped, each reading step ended could not run naming the drop, and its key and output object deleted. It takes the run's key and why.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "key": map[string]any{"type": "string"}, "why": map[string]any{"type": "string"}}, "required": []string{"key", "why"}}, + Run: func(args map[string]any) (any, error) { + in, err := as[seat.TestRunsDropInput](args) + if err != nil { + return nil, err + } + if strings.TrimSpace(in.Key) == "" || strings.TrimSpace(in.Why) == "" { + return nil, errors.New("test-runs-drop names the run's key and why") + } + return ask("seat:"+seat.Seat+"."+seat.VerbTestRunsDrop, in) + }}, + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops.go index dc1fde33..1915061c 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops.go @@ -36,7 +36,7 @@ func deleteBound(ctx context.Context, bounds jetstream.KeyValue) func(key string // armDropBound arms the five-minute bound of a drop, a key of the state bounds with its own time to live (A-77, A-222). func armDropBound(ctx context.Context, bounds jetstream.KeyValue) func(key string) error { return func(key string) error { - _, err := bounds.Create(ctx, key, []byte(`{"drop":"`+key+`"}`), jetstream.KeyTTL(pipeline.AskBound)) + _, err := bounds.Create(ctx, key, []byte(`{"drop":"`+key+`"}`), jetstream.KeyTTL(pipeline.DropBound)) if errors.Is(err, jetstream.ErrKeyExists) { return nil } @@ -67,26 +67,19 @@ func dropUnanswered(ctx context.Context, w callFindings, module string, now func return w.Write(ctx, condseat.Finding{Scope: mcc.ScopeCall, ID: id, Kind: mcc.KindCallUnanswered, Severity: condseat.FindingSeverityWarning, Summary: fmt.Sprintf("the call %s of ask %s is unanswered", call, ask), Headline: "A call whose answer is kept is not answered", - Explanation: fmt.Sprintf("The drop of the superseded step's ask %s was asked of the seat %s as a kept call and no holder answered it within %s, so the build it names may still run.", ask, buildSeat, pipeline.AskBound), + Explanation: fmt.Sprintf("The drop of the superseded step's ask %s was asked of the seat %s as a kept call and no holder answered it within %s, so the build it names may still run.", ask, buildSeat, pipeline.DropBound), Source: module + "." + mcc.KindCallUnanswered, Observer: module, At: now().UTC(), Subject: condseat.Subject{Scope: mcc.ScopeCall, ID: id}}) } } -// boundExpired takes the expiry of a key of the state bounds: a drop's says the drop unanswered, an ask's ends the step no -// holder took (§32, §535, A-77); another key is none of the runner's. -func boundExpired(h *pipeline.Hearer, unanswered func(buildSeat, ask string) error, logf func(string, ...any)) func(key string) { +// boundExpired takes the expiry of a key of the state bounds: a drop's says the drop unanswered (A-222); another key is none +// of the runner's, an ask having no bound of its own (§535, §647 as A-466y amends them). +func boundExpired(unanswered func(buildSeat, ask string) error, logf func(string, ...any)) func(key string) { return func(key string) { if s, ask, ok := pipeline.ParseDropBound(key); ok { if err := unanswered(s, ask); err != nil { logf("the drop of ask %s on %s, unanswered, is not said: %v", ask, s, err) } - return - } - if !pipeline.IsAskKey(key) { - return - } - if err := h.NotTaken(key, pipeline.AskBound); err != nil { - logf("the step of ask %s, which no holder took, is not ended: %v", key, err) } } } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops_test.go index 8b8321b0..d10bfce0 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/drops_test.go @@ -31,7 +31,7 @@ func (f *fakeFindings) Clear(_ context.Context, scope, id, kind, _ string) error func TestADropBoundsExpirySaysCallUnanswered(t *testing.T) { w := &fakeFindings{} var logged int - expired := boundExpired(nil, dropUnanswered(context.Background(), w, "the-runner", func() time.Time { return time.Unix(100, 0) }), func(string, ...any) { logged++ }) + expired := boundExpired(dropUnanswered(context.Background(), w, "the-runner", func() time.Time { return time.Unix(100, 0) }), func(string, ...any) { logged++ }) expired("not-a-bound") if len(w.written) != 0 { t.Fatalf("a key of no drop was said: %+v", w.written) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 7d973c3d..11dcd1c6 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -67,8 +67,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - // The bound of each ask a build seat's holder must take: a key with its own time to live, its expiry heard (§535, A-77). - bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound an ask no holder takes") + // The bound of each superseded step's drop: a key with its own time to live, its expiry heard (A-222, A-77). + bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound a drop no holder answers") if err != nil { return err } @@ -151,14 +151,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // for its step. starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say, Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module, - Arm: armBound(ctx, bounds), Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains), - Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has} + Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains), + Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing} placeHalf := func(ev catalog.CatalogRevisionEvent, half pipeline.Half) error { n, err := starter.PlaceRun(ev, half, rv.Catalog.View) logger.Printf("the candidate %s's revision %d placed %d pipelines of its run", ev.Group, ev.Revision, n) return err } resume := func(branch string) { + rv.Holders.refresh() // a half held may name the claimant of the seat-holders view (§647) half, ok := rv.Catalog.View(branch) if !ok { return @@ -182,6 +183,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro resumer.Store(&resume) // The view reaching a newer revision places the steps of the pipelines that waited for it, and asks nothing again (A-448, §200). advance := func(branch string) { + rv.Holders.refresh() if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 { logger.Printf("the pipelines of %s that waited for a newer revision of its half: %d placed (%v)", branch, n, err) } @@ -227,23 +229,41 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // A newer commit or publish supersedes the older running pipelines it replaces, their asks dropped as kept calls (§31, // §350; A-182, A-202). starter.Supersede = func(newer seat.Pipeline) { - n, err := hearer.Supersede(newer, pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings)), dropBound(ctx, bounds)) + n, err := hearer.Supersede(newer, pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings))) if err != nil { logger.Printf("the pipelines %s supersedes: %d ended, then %v", newer.Pipeline, n, err) } else if n > 0 { logger.Printf("the pipeline %s superseded %d older pipelines", newer.Pipeline, n) } } + // The test runs (§724, §725): one run per candidate, level and digest, kept in the state test-runs with its clock. + runDrop := pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings)) + runsKV, runObjs, err := openTestRuns(ctx, js, cred, getenv, starter, hearer, runDrop, logger.Printf) + if err != nil { + return err + } + if err := watchRunBounds(ctx, runsKV, hearer.RunExpired, logger.Printf); err != nil { + return err + } + if err := hearStepProgress(heard, hearer.RunProgress, logger.Printf); err != nil { + return err + } if err := hearStepEnded(heard, hearer, logger.Printf); err != nil { return err } if err := hearToolchain(ctx, heard, toolchains, logger.Printf); err != nil { return err } - if err := hearAskTaken(ctx, heard, bounds, hearer.Taken, logger.Printf); err != nil { + taken := func(ask, node string) error { + if err := hearer.Taken(ask, node); err != nil { + return err + } + return hearer.RunTaken(ask, node) + } + if err := hearAskTaken(heard, taken, logger.Printf); err != nil { return err } - expired := boundExpired(hearer, dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf) + expired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf) if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil { return err } @@ -261,6 +281,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } // An entry whose step ended and was not appended is appended at the start, in order (§606, §549). hearer.AppendUnlogged() + // A test run whose bound passed while the runner was away ends now, as its expiry would have ended it (§725). + if n, err := hearer.TakeUp(); err != nil || n > 0 { + logger.Printf("the test runs taken up at the start: %d ended past their bound (%v)", n, err) + } go whileTheLogIsFull(ctx, fills, hearer, time.Minute) // The seat list, read over the calls connection and heard on the hub (seatlist.Part), which the seat's write verbs check // their callers against (A-186): registered before the hub starts, read again once its events are live. @@ -273,7 +297,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // mesh-wants), never on EVENTS: bound here, or no step of any pipeline is ever heard ending (G-1 §24). logger.Printf("the kept seat events heard: %d streams bound", hub.StartKept(ctx, js, cred)) // The seat's write verbs are answered here, by the process that holds the pipelines (A-107). - stopVerbs, err := answerVerbs(calls, seats, writeVerbs(ctx, hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls, bounds)) + verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls) + verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill} + stopVerbs, err := answerVerbs(calls, seats, verbs) if err != nil { return err } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go index 25489f6a..32ee6038 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go @@ -65,8 +65,11 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes t.Fatal(err) } } - if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_bounds", History: 1, LimitMarkerTTL: time.Minute}); err != nil { - t.Fatal(err) + // bounds and test-runs keep keys with their own time to live (§725), as the bus makes them from the manifest's key-ttl. + for _, state := range []string{"bounds", TestRunsState} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil { + t.Fatal(err) + } } // The step log, the runner's old logs section (§470): the stream the bus makes for it. if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go index ee7d7ded..d19683fe 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -114,7 +114,11 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any)) _ = m.Term() return } - _, err = h.OnStepEnded(bs, m.Subject(), ev) + var shared []seat.Shared + if bs.Shared != nil { + shared = bs.Shared(m.Data()) + } + _, err = h.OnStepEndedShared(bs, m.Subject(), ev, shared) switch { case errors.Is(err, pipeline.ErrRefused): logf("%v", err) @@ -133,11 +137,10 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any)) return nil } -// hearAskTaken deletes the bound of each ask a build seat's holder took (§602, §647) and keeps on its step the node that took -// it (A-182): the ask is answered, so its five minutes end with no finding. -func hearAskTaken(ctx context.Context, hub handlers, bounds jetstream.KeyValue, taken func(ask, node string) error, logf func(string, ...any)) error { +// hearAskTaken keeps on its step the node whose build seat holder took an ask (§602, §647; A-182), so a drop of it is asked of +// that node's holder, and begins the clock of a test run's ask (§725). +func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(string, ...any)) error { for _, ts := range pipeline.TakenSeats { - ts := ts err := hub.Handle(ts.Subject, func(m jetstream.Msg) { var ev nodebuildgo.AskTaken if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" { @@ -145,12 +148,6 @@ func hearAskTaken(ctx context.Context, hub handlers, bounds jetstream.KeyValue, _ = m.Term() return } - if err := bounds.Delete(ctx, ev.Ask); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { - logf("the bound of ask %s, taken on %s, is not deleted, heard again: %v", ev.Ask, ev.Node, err) - _ = m.Nak() - return - } - // The step keeps the node that took its ask, so a drop of it is asked of that node's holder (A-182). if err := taken(ev.Ask, ev.Node); err != nil { logf("the node %s that took ask %s is not kept on its step, heard again: %v", ev.Node, ev.Ask, err) _ = m.Nak() @@ -165,19 +162,8 @@ func hearAskTaken(ctx context.Context, hub handlers, bounds jetstream.KeyValue, return nil } -// armBound is the bound of an ask: a key of the state bounds with its own time to live, named for the ask. -func armBound(ctx context.Context, bounds jetstream.KeyValue) pipeline.Arm { - return func(ask string) error { - _, err := bounds.Create(ctx, ask, []byte(`{"ask":"`+ask+`"}`), jetstream.KeyTTL(pipeline.AskBound)) - if errors.Is(err, jetstream.ErrKeyExists) { - return nil - } - return err - } -} - -// watchBounds hears the end of each ask's bound, the bus's marker of the key's expiry, and ends the step no holder took (§32, -// §535, A-77), and says a drop no holder answered unanswered (A-222): no timer in the process. +// watchBounds hears the end of each drop's bound, the bus's marker of the key's expiry, and says a drop no holder answered +// unanswered (A-222, A-77): no timer in the process. func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error { w, err := bounds.WatchAll(ctx, jetstream.UpdatesOnly()) if err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go new file mode 100644 index 00000000..22a724d1 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go @@ -0,0 +1,416 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage" + nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// TestRunsState is the runner's state of its test runs (design 47 G §725), declared in its manifest's old state section. +const TestRunsState = "test-runs" + +// testRunsKV is the state test-runs on the bus: its runs by compare-and-set, and its bound keys, each written with its own +// time to live (A-77, A-470). +type testRunsKV struct { + ctx context.Context + js jetstream.JetStream + kv jetstream.KeyValue +} + +func (t testRunsKV) Create(key string, r pipeline.Run) error { + raw, err := json.Marshal(r) + if err != nil { + return err + } + if _, err := t.kv.Create(t.ctx, key, raw); errors.Is(err, jetstream.ErrKeyExists) { + return pipeline.ErrExists + } else if err != nil { + return err + } + return nil +} + +func (t testRunsKV) Get(key string) (pipeline.Run, uint64, error) { + e, err := t.kv.Get(t.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return pipeline.Run{}, 0, pipeline.ErrNoRun + } + if err != nil { + return pipeline.Run{}, 0, err + } + var r pipeline.Run + if err := json.Unmarshal(e.Value(), &r); err != nil { + return pipeline.Run{}, 0, fmt.Errorf("the test run %s is no test run: %w", key, err) + } + return r, e.Revision(), nil +} + +func (t testRunsKV) Update(key string, r pipeline.Run, revision uint64) (uint64, error) { + raw, err := json.Marshal(r) + if err != nil { + return 0, err + } + rev, err := t.kv.Update(t.ctx, key, raw, revision) + if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) { + return 0, pipeline.ErrChanged + } + return rev, err +} + +func (t testRunsKV) Delete(key string, revision uint64) error { + err := t.kv.Delete(t.ctx, key, jetstream.LastRevision(revision)) + if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) { + return pipeline.ErrChanged + } + if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err + } + return nil +} + +// deleteBound deletes a bound key, whose value is no run: a key gone is no fault. +func (t testRunsKV) deleteBound(key string) error { + if err := t.kv.Delete(t.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err + } + return nil +} + +// boundValue is a bound key's value: the key and the ask of the run it bounds (A-470n). +type boundValue struct { + Bound string `json:"bound"` + Ask string `json:"ask"` +} + +// Bound writes key again holding ask, with ttl as its own time to live, published at the key's latest revision (a delete +// marker counted), so a write between is read again and never overwritten unseen (A-470n); the earlier message, replaced, +// takes its time to live with it, so the expiry is heard only ttl after the last write. +func (t testRunsKV) Bound(key, ask string, ttl time.Duration) (uint64, error) { + raw, err := json.Marshal(boundValue{Bound: key, Ask: ask}) + if err != nil { + return 0, err + } + subject := "$KV." + t.kv.Bucket() + "." + key + for { + var last uint64 + st, err := t.js.Stream(t.ctx, "KV_"+t.kv.Bucket()) + if err != nil { + return 0, err + } + m, err := st.GetLastMsgForSubject(t.ctx, subject) + switch { + case err == nil: + last = m.Sequence + case !errors.Is(err, jetstream.ErrMsgNotFound): + return 0, err + } + ack, err := t.js.Publish(t.ctx, subject, raw, jetstream.WithMsgTTL(ttl), jetstream.WithExpectLastSequencePerSubject(last)) + if err != nil && strings.Contains(err.Error(), "wrong last sequence") { + continue + } + if err != nil { + return 0, err + } + return ack.Sequence, nil + } +} + +// Bounded reads a bound key: the ask it holds and its revision. +func (t testRunsKV) Bounded(key string) (string, uint64, bool, error) { + e, err := t.kv.Get(t.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return "", 0, false, nil + } + if err != nil { + return "", 0, false, err + } + var v boundValue + _ = json.Unmarshal(e.Value(), &v) + return v.Ask, e.Revision(), true, nil +} + +func (t testRunsKV) Unbound(key string) error { return t.deleteBound(key) } + +// UnboundAt deletes a bound key at revision: a later write of it is kept. +func (t testRunsKV) UnboundAt(key string, revision uint64) error { + err := t.kv.Delete(t.ctx, key, jetstream.LastRevision(revision)) + if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence") || errors.Is(err, jetstream.ErrKeyNotFound)) { + return nil + } + return err +} + +func (t testRunsKV) Has(key string) (bool, error) { + _, err := t.kv.Get(t.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return false, nil + } + return err == nil, err +} + +func (t testRunsKV) Keys() ([]string, error) { + all, err := t.kv.Keys(t.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return all, err +} + +// runsKept is the state test-runs as the verb test-runs reads it (A-470i): its keys, and each key's value as written. +type runsKept struct { + ctx context.Context + kv jetstream.KeyValue +} + +func (r runsKept) Keys() ([]string, error) { + ks, err := r.kv.Keys(r.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return ks, err +} + +func (r runsKept) Value(key string) (json.RawMessage, error) { + e, err := r.kv.Get(r.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return e.Value(), nil +} + +// runObjects is the state test-runs' own object store, `test-runs-objects`, made with the state (A-469e), opened when +// first used. +type runObjects struct { + ctx context.Context + js jetstream.JetStream + module string + mu sync.Mutex + store jetstream.ObjectStore +} + +func (o *runObjects) open() (jetstream.ObjectStore, error) { + o.mu.Lock() + defer o.mu.Unlock() + if o.store != nil { + return o.store, nil + } + s, err := o.js.ObjectStore(o.ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(TestRunsState))) + if err != nil { + return nil, fmt.Errorf("the object store of the state %s is not open: %w", TestRunsState, err) + } + o.store = s + return s, nil +} + +func (o *runObjects) Put(name string, data []byte) error { + s, err := o.open() + if err != nil { + return err + } + _, err = s.PutBytes(o.ctx, name, data) + return err +} + +func (o *runObjects) Get(name string) ([]byte, error) { + s, err := o.open() + if err != nil { + return nil, err + } + data, err := s.GetBytes(o.ctx, name) + if errors.Is(err, jetstream.ErrObjectNotFound) { + return nil, pipeline.ErrNoRun + } + return data, err +} + +func (o *runObjects) Delete(name string) error { + s, err := o.open() + if err != nil { + return err + } + if err := s.Delete(o.ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) { + return err + } + return nil +} + +// runsFill reads test-runs' fill, its key-value stream's bytes (A-469d): judged by the process against its cap setting. +func runsFill(ctx context.Context, js jetstream.JetStream, module string, now func() time.Time) func() (seat.Fill, error) { + return func() (seat.Fill, error) { + st, err := js.Stream(ctx, "KV_"+ownBucket(module, TestRunsState)) + if err != nil { + return seat.Fill{}, err + } + info, err := st.Info(ctx) + if err != nil { + return seat.Fill{}, err + } + at := now().UTC() + return seat.Fill{Store: TestRunsState, Bytes: int64(info.State.Bytes), Cap: info.Config.MaxBytes, At: &at}, nil + } +} + +// The files the manifest writes each setting of the test runs into, `${config:}`, named by the environment (A-469x). +const ( + envRunsCap = "MESH_RUNNER_TEST_RUNS_CAP_FILE" + envRunsKept = "MESH_RUNNER_TEST_RUNS_KEPT_FILE" + envStepBound = "MESH_RUNNER_TEST_STEP_BOUND_FILE" + envUntakenBound = "MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE" +) + +// maxRunsKept is test-runs-kept's maximum (A-470a): the pipeline naming a run's key is kept at least that long. +const maxRunsKept = 168 * time.Hour + +// runSettings reads the settings of the test runs at the start (§725; A-469x): test-runs-cap-mib a whole number of MiB, +// test-runs-kept, test-step-bound and run-untaken-bound Go durations. A setting whose file is missing or empty is its +// default; a content that is neither is an error naming the file. +func runSettings(getenv func(string) string) (pipeline.RunSettings, error) { + s := pipeline.DefaultRunSettings + if v, err := settingText(getenv(envRunsCap)); err != nil { + return s, err + } else if v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 1 { + return s, fmt.Errorf("%s holds %q, which is not a positive whole number", getenv(envRunsCap), v) + } + s.CapBytes = int64(n) << 20 + } + for _, d := range []struct { + env string + to *time.Duration + }{{envRunsKept, &s.Kept}, {envStepBound, &s.StepBound}, {envUntakenBound, &s.Untaken}} { + v, err := settingText(getenv(d.env)) + if err != nil { + return s, err + } + if v == "" { + continue + } + t, err := time.ParseDuration(v) + if err != nil || t <= 0 { + return s, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(d.env), v) + } + *d.to = t + } + if s.Kept > maxRunsKept { + return s, fmt.Errorf("%s holds %s, past test-runs-kept's maximum %s (A-470a)", getenv(envRunsKept), s.Kept, maxRunsKept) + } + return s, nil +} + +// settingText is one setting file's content, trimmed: empty for no path, no file or an empty one. +func settingText(path string) (string, error) { + if path == "" { + return "", nil + } + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return "", nil + } + if err != nil { + return "", err + } + return strings.TrimSpace(string(raw)), nil +} + +// progressSubjects hear every build seat's step-progress (§725), from their contracts. +var progressSubjects = []string{nodebuildgo.SubjectStepProgressAll, nodebuildnode.SubjectStepProgressAll, nodebuildimage.SubjectStepProgressAll} + +// hearStepProgress begins again the clock of the test run whose ask a build seat's holder says progress of (§725, A-466p). +func hearStepProgress(hub handlers, progress func(ask string) error, logf func(string, ...any)) error { + for _, subject := range progressSubjects { + err := hub.Handle(subject, func(m jetstream.Msg) { + var ev nodebuildgo.StepProgress + if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" { + logf("%s is no step-progress: %v", m.Subject(), err) + _ = m.Term() + return + } + if err := progress(ev.Ask); err != nil { + logf("the clock of the test run of ask %s is not begun again, heard again: %v", ev.Ask, err) + _ = m.Nak() + return + } + _ = m.Ack() + }) + if err != nil { + return err + } + } + return nil +} + +// watchRunBounds hears the expiry of each test run's bound key, the bus's marker of the key's end (§725): no timer in the +// process. +func watchRunBounds(ctx context.Context, kv jetstream.KeyValue, expired func(key string) error, logf func(string, ...any)) error { + w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) + if err != nil { + return err + } + go func() { + for { + select { + case <-ctx.Done(): + _ = w.Stop() + return + case e, ok := <-w.Updates(): + if !ok { + return + } + if e == nil || e.Operation() != jetstream.KeyValuePurge || !pipeline.IsRunBound(e.Key()) { + continue + } + if err := expired(e.Key()); err != nil { + logf("the test run of the bound %s that ended is not ended: %v", e.Key(), err) + } + } + } + }() + return nil +} + +// openTestRuns opens the state test-runs and wires it into the starter, the hearer and the ager (§725). +func openTestRuns(ctx context.Context, js jetstream.JetStream, cred process.Account, getenv func(string) string, + starter *pipeline.Starter, hearer *pipeline.Hearer, drop pipeline.Drop, logf func(string, ...any)) (jetstream.KeyValue, *runObjects, error) { + settings, err := runSettings(getenv) + if err != nil { + return nil, nil, err + } + kv, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, TestRunsState), "share a test run") + if err != nil { + return nil, nil, err + } + runs := testRunsKV{ctx: ctx, js: js, kv: kv} + objects := &runObjects{ctx: ctx, js: js, module: cred.Module} + starter.TestRuns, starter.Clock, starter.Runs = runs, runs, settings + starter.RunsFill = runsFill(ctx, js, cred.Module, time.Now) + starter.EndReaders = hearer.EndReaders + hearer.TestRuns, hearer.Clock, hearer.Runs, hearer.DropRun = runs, runs, settings, drop + hearer.RunOutputs = objects + if hearer.Ager != nil { + hearer.Ager.Runs, hearer.Ager.RunOutputs, hearer.Ager.RunsKept = runs, objects.Delete, settings.Kept + } + logf("test runs: cap %d bytes, kept %s, step bound %s, untaken bound %s", settings.CapBytes, settings.Kept, settings.StepBound, settings.Untaken) + return kv, objects, nil +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go new file mode 100644 index 00000000..6ec49142 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go @@ -0,0 +1,536 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/seatlist" + "git.novox.be/novox/mesh-sdk/go/testbus" + "git.novox.be/novox/mesh-sdk/go/views" + "git.novox.be/novox/mesh-sdk/go/views/viewstest" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// Test 109 (design 47 G §733 *Whole (runner…)*, §724 to §728, §535 and §647 as A-466y and A-466z amend them; A-468f: in +// mesh-delivery with a fixture build seat holder; A-470f: no reader-only refusal). The runner whole against the real bus: a +// module whose program and tools bundle test one directory asks one unit and one integration run, both artifacts' steps +// ending with its result and shared; a failing test fails once and is counted once; a rerun of one artifact's failed step +// that passes moves the other's, the pipeline ending again, and an accepted step does not move; a runner restarted during a +// run takes it up by its ask, its clock holding that ask (A-470n); a test run waiting with a holder present is not failed; +// a replaced reader leaves the ask to the other readers; a run that keeps saying its progress past its bound is not ended, +// and a taken ask silent for its bound ends could not run with its readers, its ask dropped; a second revision with the +// same digests asks its own runs; a seat the seat-holders view names no holder for ends its pipeline failed at once, its +// other steps not asked (A-470g, A-470h); the process answers test-runs with its fill against test-runs-cap-mib and +// test-runs-drop deletes the key and its output object (A-470i); at the cap a step ends could not run, naming the fill. +// The step bound is two seconds here, as the setting test-step-bound names it, where the mesh's is 30 minutes. +func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { + url := testbus.URL(t) + nc, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + const module = "mesh-delivery-pipeline" + for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries"} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil { + t.Fatal(err) + } + } + for _, state := range []string{"bounds", TestRunsState} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil { + t.Fatal(err) + } + } + // test-runs' own object store, made with the state (A-469e). + if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName(module, catalogue.StateObjectsName(TestRunsState))}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil { + t.Fatal(err) + } + events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>", "mesh.stream.*.event.>"}}) + if err != nil { + t.Fatal(err) + } + if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: "n1_" + module, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil { + t.Fatal(err) + } + + // The seat-holders view of the seat mesh-module-assigner's claimant (§647, A-466z), served as its owner does: node-build-go + // has a holder, node-build-node none. + holders := viewstest.Serve(t, ctx, js, nc, "assigner", "seat-holders", "assigner.seat-holders", "n1_"+module) + if _, err := holders.Put(ctx, "node-build-go", []byte(`{"holders":[{"module":"mesh-build-go"}]}`)); err != nil { + t.Fatal(err) + } + + // The registrations: app tests its program and tools bundle from one directory, its unit step required and its + // integration step optional; web is built by node-build-node. + policy := catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required", Integration: "optional", Composition: "bypass", Review: "bypass", Approval: "bypass"}} + regs := []catalog.Registration{{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: policy}, + {Module: "web", Branches: []string{"main"}, Origin: "o/web", Policy: policy}} + answer := func(subject string, body any) { + if _, err := nc.Subscribe(subject, func(m *nats.Msg) { + raw, _ := json.Marshal(map[string]any{"result": body}) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + } + answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: regs}) + answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{}) + entries := []catalog.CatalogEntry{ + {Module: "app", Commit: "c1", Fp: "src2:aa", Path: "modules/app", Repository: "o/app", + Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + {Module: "web", Commit: "w1", Fp: "src2:ww", Path: "modules/web", Repository: "o/web", Artifacts: []catalog.Artifact{{Ref: "web:program", Language: "node"}}}, + {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, + {Module: "mesh-build-node", Acts: []catalog.ActingRule{{Seat: "node-build-node", Step: "build", On: "node"}}}, + {Module: "assigner", Manifest: map[string]any{"module": "assigner", "version": "0.1", "claims": []any{map[string]any{"name": seatlist.HolderSeat, "scope": "mesh"}}}}, + } + var revision atomic.Int64 + revision.Store(1) + if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call.revision.*", func(m *nats.Msg) { + r := revision.Load() + raw, _ := json.Marshal(map[string]any{"result": catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: r, Kind: "trunk", Branch: "main", Source: 2 + r}, Entries: entries}}) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + def, err := os.ReadFile("../../../mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json") + if err != nil { + t.Fatal(err) + } + if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call."+seatlist.Verb+".*", func(m *nats.Msg) { + var in struct { + Seat string `json:"seat"` + } + _ = json.Unmarshal(m.Data, &in) + entry := seatlist.Entry{Seat: seat.Seat, Source: seatlist.SourceTrunk, Digest: "sha256:runner"} + var result any = seatlist.Listing{Revision: 1, Seats: []seatlist.Entry{entry}} + if in.Seat != "" { + entry.Definition = def + result = seatlist.Answer{Seat: in.Seat, Found: in.Seat == seat.Seat, Entry: &entry, Revision: 1} + } + raw, _ := json.Marshal(map[string]any{"result": result}) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + // Every pipeline-ended said, and every drop asked of a build seat. + var mu sync.Mutex + var endings []seat.PipelineEnded + var drops []string + if _, err := nc.Subscribe("mesh.seat.mesh-pipeline-runner.event.pipeline-ended.>", func(m *nats.Msg) { + var e seat.PipelineEnded + if json.Unmarshal(m.Data, &e) == nil { + mu.Lock() + endings = append(endings, e) + mu.Unlock() + } + }); err != nil { + t.Fatal(err) + } + if _, err := nc.Subscribe("mesh.seat.*.call.drop.>", func(m *nats.Msg) { + mu.Lock() + drops = append(drops, string(m.Data)) + mu.Unlock() + raw, _ := json.Marshal(map[string]any{"result": map[string]any{}}) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + ended := func(key string) []seat.PipelineEnded { + mu.Lock() + defer mu.Unlock() + var out []seat.PipelineEnded + for _, e := range endings { + if e.Pipeline == key { + out = append(out, e) + } + } + return out + } + + // The settings, as the manifest's resources write them (A-469x): a cap of 1 MiB, a step bound of two seconds. + dir := t.TempDir() + files := map[string]string{envRunsCap: "1", envStepBound: "2s"} + env := map[string]string{} + for name, content := range files { + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + env[name] = path + } + broker := filepath.Join(dir, "broker") + raw, _ := json.Marshal(process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"}) + if err := os.WriteFile(broker, raw, 0o600); err != nil { + t.Fatal(err) + } + env[process.BrokerFileEnv] = broker + logs := &syncBuffer{} + start := func() func() { + runCtx, stop := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- run(runCtx, func(k string) string { return env[k] }, logs) }() + return func() { stop(); <-done } + } + stop := start() + defer func() { stop() }() + + pipelines, err := js.KeyValue(ctx, module+"_pipelines") + if err != nil { + t.Fatal(err) + } + runsKV, err := js.KeyValue(ctx, module+"_"+TestRunsState) + if err != nil { + t.Fatal(err) + } + firstViews, err := js.KeyValue(ctx, module+"_"+views.StatusState) + if err != nil { + t.Fatal(err) + } + // The seat-holders view's first view is read before any pipeline is placed, so a seat is judged from it (A-466y). + waitFor(t, ctx, logs, func() bool { + e, err := firstViews.Get(ctx, views.StatusKey("assigner", "seat-holders")) + var s views.Status + return err == nil && json.Unmarshal(e.Value(), &s) == nil && !s.Answered.IsZero() + }) + + takeIn := func(module, commit, fp string, source, rev int64) { + taken, _ := json.Marshal(catalog.TakeInTaken{Key: "m." + commit, Kind: "taken", Branch: "main", Source: source, Revision: rev, + Entries: []catalog.TakeInEntry{{Module: module, Commit: commit, Fingerprint: fp}}}) + if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", taken); err != nil { + t.Fatal(err) + } + } + placed := func(key string) seat.Pipeline { + var p seat.Pipeline + waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && len(p.Steps) > 1 }) + return p + } + read := func(key string) seat.Pipeline { + var p seat.Pipeline + readPipeline(ctx, pipelines, key, &p) + return p + } + step := func(p seat.Pipeline, artifact, name string) seat.StepResult { + for _, s := range p.Steps { + if s.Artifact == artifact && s.Step == name { + return s + } + } + return seat.StepResult{} + } + runOf := func(key string) (pipeline.Run, bool) { + e, err := runsKV.Get(ctx, key) + var r pipeline.Run + return r, err == nil && json.Unmarshal(e.Value(), &r) == nil + } + runKeys := func(candidate string) []string { + ks, _ := runsKV.Keys(ctx) + var out []string + for _, k := range ks { + if !pipeline.IsRunBound(k) && strings.HasPrefix(k, candidate+".") { + out = append(out, k) + } + } + return out + } + runOfStep := func(key, artifact, name string) (string, pipeline.Run) { + var run string + var r pipeline.Run + waitFor(t, ctx, logs, func() bool { + s := step(read(key), artifact, name) + if len(s.Shared) == 0 { + return false + } + run = s.Shared[0].Run + got, ok := runOf(run) + r = got + return ok && r.Ask != "" + }) + return run, r + } + publish := func(subject string, body any) { + raw, _ := json.Marshal(body) + if _, err := js.Publish(ctx, subject, raw); err != nil { + t.Fatal(err) + } + } + taken := func(ask string) { + publish(nodebuildgo.SubjectAskTaken("n1"), nodebuildgo.AskTaken{Ask: ask, Asker: module, Node: "n1", At: time.Now()}) + } + endAsk := func(key, ask, name, outcome string, tests *nodebuildgo.TestCounts) { + now := time.Now() + publish(nodebuildgo.SubjectStepEnded("n1"), nodebuildgo.StepEnded{ID: ask, Ask: ask, Asker: module, Pipeline: key, Module: "app", Commit: "c1", + Fingerprint: "src2:aa", Step: name, Node: "n1", Outcome: outcome, Tests: tests, At: now, Ended: &now}) + } + passBuild := func(key, module, commit, fp string) { + ask := step(read(key), "", seat.StepResultStepBuild).Ask + publish("mesh.seat.mesh-builds.event.step-ended.n1", builds.StepEnded{ID: ask, Pipeline: key, Step: "build", Node: "n1", Outcome: "passed", + Entries: []builds.BuildEntry{{Module: module, Commit: commit, Fingerprint: fp}}}) + } + call := func(verb string, in, out any) string { + var reply struct { + Result json.RawMessage `json:"result"` + Error string `json:"error"` + } + waitFor(t, ctx, logs, func() bool { + m, err := nc.Request("mesh.seat.mesh-pipeline-runner.call."+verb+".agent~dev", mustJSON(in), 2*time.Second) + reply.Error = "" + return err == nil && json.Unmarshal(m.Data, &reply) == nil && !strings.Contains(reply.Error, "not yet read") + }) + if out != nil && reply.Error == "" { + _ = json.Unmarshal(reply.Result, out) + } + return reply.Error + } + + // One unit run and one integration run for both artifacts (§724), each read by both, waiting for a build node until its + // ask is taken (A-469b), its holder present and nothing failing it (A-466y). + const p3 = "main.3.app" + takeIn("app", "c1", "src2:aa", 3, 1) + passBuild(p3, "app", "c1", "src2:aa") + unitRun, unit := runOfStep(p3, "app:program", "unit") + integRun, integ := runOfStep(p3, "app:program", "integration") + if got := runKeys("main.1"); len(got) != 2 || len(unit.Readers) != 2 || len(integ.Readers) != 2 { + t.Fatalf("runs %v, unit readers %+v, integration readers %+v; want one run per level read by both artifacts", got, unit.Readers, integ.Readers) + } + if s := step(read(p3), "app:tools", "unit"); s.Outcome != seat.StepResultOutcomeWaitingForABuildNode || s.Ask != "" || s.Shared[0].Ask != unit.Ask { + t.Fatalf("the tools' unit step = %+v; want it waiting for a build node, reading the program's run", s) + } + + // A runner restarted during a run takes it up again by its ask (§725): the run's end, heard after the restart, ends both. + taken(unit.Ask) + waitFor(t, ctx, logs, func() bool { r, _ := runOf(unitRun); return r.Node == "n1" }) + // Its clock holds the run's ask (A-470n). + waitFor(t, ctx, logs, func() bool { + e, err := runsKV.Get(ctx, pipeline.BoundPrefix+unitRun) + return err == nil && strings.Contains(string(e.Value()), `"ask":"`+unit.Ask+`"`) + }) + stop() + stop = start() + // It keeps saying its progress past its bound and is not ended (§725, §720). + // Each progress is heard before the next is said: the run's clock is written again (its bound key's revision rises). + boundRev := func() uint64 { + e, err := runsKV.Get(ctx, pipeline.BoundPrefix+unitRun) + if err != nil { + return 0 + } + return e.Revision() + } + for at := time.Now(); time.Since(at) < 3*time.Second; { + before := boundRev() + publish(nodebuildgo.SubjectStepProgress("n1"), nodebuildgo.StepProgress{Ask: unit.Ask, Asker: module, Phase: "run", At: time.Now()}) + waitFor(t, ctx, logs, func() bool { return boundRev() > before }) + } + if r, _ := runOf(unitRun); r.Ended != nil { + t.Fatalf("a run saying its progress past its bound ended: %+v", r) + } + // A failing test fails once and is counted once (§726). + endAsk(p3, unit.Ask, "unit", "failed", &nodebuildgo.TestCounts{Unit: 5}) + taken(integ.Ask) + endAsk(p3, integ.Ask, "integration", "failed", &nodebuildgo.TestCounts{Integration: 2}) + waitFor(t, ctx, logs, func() bool { return len(ended(p3)) == 1 }) + p := read(p3) + for _, a := range []string{"app:program", "app:tools"} { + if s := step(p, a, "unit"); s.Outcome != seat.StepResultOutcomeFailed || len(s.Shared) == 0 || s.Shared[0].Run != unitRun { + t.Errorf("%s's unit step = %+v; want failed with the run in shared", a, s) + } + } + for _, k := range []string{unitRun, integRun} { + if r, _ := runOf(k); r.Ended == nil || r.Outcome != seat.StepResultOutcomeFailed { + t.Errorf("the run %s at its end = %+v; want it ended failed", k, r) + } + } + if e := ended(p3)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Tests == nil || e.Tests.Unit != 5 || e.Tests.Integration != 2 { + t.Fatalf("pipeline-ended = %+v; want failed, the tests counted once", e) + } + + // An accepted step does not move; a rerun of the program's unit step that passes moves the tools' to passed, and the + // pipeline says pipeline-ended again (§727, A-470d). + if e := call(seat.VerbAccept, seat.AcceptInput{Pipeline: p3, Step: "integration", Artifact: "app:program", Why: "flaky"}, nil); e != "" { + t.Fatalf("accept: %s", e) + } + if e := call(seat.VerbRerun, seat.RerunInput{Pipeline: p3, Step: "unit", Artifact: "app:program", Why: "fixed"}, nil); e != "" { + t.Fatalf("rerun: %s", e) + } + var again pipeline.Run + waitFor(t, ctx, logs, func() bool { r, ok := runOf(unitRun); again = r; return ok && r.Ended == nil && r.Ask != unit.Ask }) + taken(again.Ask) + endAsk(p3, again.Ask, "unit", "passed", &nodebuildgo.TestCounts{Unit: 5}) + if e := call(seat.VerbRerun, seat.RerunInput{Pipeline: p3, Step: "integration", Artifact: "app:tools", Why: "fixed"}, nil); e != "" { + t.Fatalf("rerun: %s", e) + } + var integAgain pipeline.Run + waitFor(t, ctx, logs, func() bool { + r, ok := runOf(integRun) + integAgain = r + return ok && r.Ended == nil && r.Ask != integ.Ask + }) + taken(integAgain.Ask) + endAsk(p3, integAgain.Ask, "integration", "passed", nil) + waitFor(t, ctx, logs, func() bool { + e := ended(p3) + return len(e) >= 2 && e[len(e)-1].Outcome == seat.PipelineEndedOutcomePassed + }) + p = read(p3) + if s := step(p, "app:tools", "unit"); s.Outcome != seat.StepResultOutcomePassed || !s.Rerun { + t.Errorf("the tools' unit step = %+v; want moved to passed, marked a rerun", s) + } + if s := step(p, "app:program", "integration"); s.Outcome != seat.StepResultOutcomeAccepted { + t.Errorf("the accepted step = %+v; want it accepted still", s) + } + for _, k := range []string{unitRun, integRun} { + if r, _ := runOf(k); r.Ended == nil || r.Outcome != seat.StepResultOutcomePassed { + t.Errorf("the run %s after its rerun = %+v; want it ended passed", k, r) + } + } + if e := ended(p3); e[len(e)-1].EndCount <= e[0].EndCount { + t.Errorf("pipeline-ended counts %d then %d; want a higher end count", e[0].EndCount, e[len(e)-1].EndCount) + } + + // A second revision with the same digests asks its own runs, so a failed test runs again (§724). + revision.Store(2) + publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 2, Kind: "trunk", Branch: "main", Source: 4}) + const p4, p5 = "main.4.app", "main.5.app" + takeIn("app", "c1", "src2:aa", 4, 2) + placed(p4) + run4, r4 := runOfStep(p4, "app:program", "integration") + if !strings.HasPrefix(run4, "main.2.") || r4.Ask == integ.Ask { + t.Fatalf("revision 2's integration run %s (%s); want its own", run4, r4.Ask) + } + // A replaced reader leaves the ask to the other readers (§726): a second pipeline of the same candidate and digest reads + // the run, the first is cancelled and leaves it, and the run's ask is not dropped. + takeIn("app", "c1", "src2:aa", 5, 2) + placed(p5) + waitFor(t, ctx, logs, func() bool { r, _ := runOf(run4); return len(r.Readers) == 4 }) + if e := call(seat.VerbCancel, seat.CancelInput{Pipeline: p4, Why: "replaced"}, nil); e != "" { + t.Fatalf("cancel: %s", e) + } + waitFor(t, ctx, logs, func() bool { + r, _ := runOf(run4) + return len(r.Readers) == 2 && r.Readers[0].Pipeline == p5 && r.Readers[1].Pipeline == p5 + }) + if r, _ := runOf(run4); r.Ended != nil || r.Ask != r4.Ask { + t.Fatalf("the run of the replaced readers = %+v; want it running under its ask", r) + } + mu.Lock() + dropped := append([]string(nil), drops...) + mu.Unlock() + if len(dropped) != 0 { + t.Fatalf("drops asked %v; want none while a reader is left\n%s", dropped, logs.String()) + } + // A taken ask silent for its bound ends could not run with its readers, its ask dropped (§725, §720). + taken(r4.Ask) + waitFor(t, ctx, logs, func() bool { + return step(read(p5), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun && + step(read(p5), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun + }) + if r, _ := runOf(run4); r.Ended == nil || r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "test-step-bound") { + t.Errorf("the silent run = %+v; want it ended could not run, naming its bound", r) + } + waitFor(t, ctx, logs, func() bool { + mu.Lock() + defer mu.Unlock() + return len(drops) == 1 && strings.Contains(drops[0], r4.Ask) + }) + + // A seat the seat-holders view names no holder for ends the pipeline failed at once, naming it, its other steps not + // asked (§535, §647; A-470g, A-470h). + const web = "main.5.web" + takeIn("web", "w1", "src2:ww", 5, 2) + waitFor(t, ctx, logs, func() bool { return len(ended(web)) == 1 }) + if e := ended(web)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Why != "the seat node-build-node has no holder: the seat-holders view names none" { + t.Fatalf("pipeline-ended = %+v; want failed at once naming the seat", e) + } + // The steps of the seat with no holder end could not run, a bypassed step stays bypassed, every other ends not asked, + // and none is asked (A-470h). + for _, s := range read(web).Steps { + want := seat.StepResultOutcomeNotAsked + switch { + case s.Status == seat.StepResultStatusBypass: + want = seat.StepResultOutcomeBypassed + case s.Seat == "node-build-node": + want = seat.StepResultOutcomeCouldNotRun + } + if s.Outcome != want || s.Ask != "" { + t.Errorf("the step %s %s of the pipeline with no holder = %s (ask %q); want %s", s.Artifact, s.Step, s.Outcome, s.Ask, want) + } + } + + // test-runs is answered by the process, its fill against test-runs-cap-mib (A-470i), each run's packages from its output + // object; test-runs-drop deletes the key and its object (§728). + var runs seat.TestRunsAnswer + if e := call(seat.VerbTestRuns, seat.TestRunsInput{Candidate: "main.1"}, &runs); e != "" || len(runs.Runs) != 2 || runs.Fill.Cap != 1<<20 || runs.Fill.Store != TestRunsState { + t.Fatalf("test-runs = %+v (%s); want main.1's two runs, the fill against 1 MiB", runs, e) + } + objects, err := js.ObjectStore(ctx, catalogue.BusName(module, catalogue.StateObjectsName(TestRunsState))) + if err != nil { + t.Fatal(err) + } + if _, err := objects.GetInfo(ctx, unitRun); err != nil { + t.Fatalf("the unit run's output object: %v", err) + } + var drop seat.TestRunsDropAnswer + if e := call(seat.VerbTestRunsDrop, seat.TestRunsDropInput{Key: unitRun, Why: "test 109"}, &drop); e != "" || drop.Run.Key != unitRun { + t.Fatalf("test-runs-drop = %+v (%s)", drop, e) + } + if _, ok := runOf(unitRun); ok { + t.Error("the dropped run's key is kept") + } + if _, err := objects.GetInfo(ctx, unitRun); err == nil { + t.Error("the dropped run's output object is kept") + } + + // At the cap a step ends could not run, naming the fill (§725, A-469d). + pad := make([]byte, 600<<10) + for _, k := range []string{"pad.a", "pad.b"} { + if _, err := runsKV.Put(ctx, k, pad); err != nil { + t.Fatal(err) + } + } + revision.Store(3) + publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 5}) + const p6 = "main.6.app" + takeIn("app", "c1", "src2:aa", 6, 3) + waitFor(t, ctx, logs, func() bool { + s := step(read(p6), "app:program", "unit") + return s.Outcome == seat.StepResultOutcomeCouldNotRun && strings.Contains(s.Why, "test-runs-cap-mib") + }) +} + +func mustJSON(v any) []byte { + raw, _ := json.Marshal(v) + return raw +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_test.go new file mode 100644 index 00000000..c8f14d81 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +// The test runs' settings reach the process through the files the manifest writes (design 47 G §725; A-469x, A-470a): the +// cap a whole number of MiB, the times Go durations, a missing file its default, a content that is neither, or a kept time +// past 168h, an error naming the file. +func TestTheTestRunsSettingsAreReadFromTheirFiles(t *testing.T) { + dir := t.TempDir() + write := func(name, content string) string { + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return p + } + rows := []struct { + name string + files map[string]string + ok bool + check func(t *testing.T, cap int64, kept, step, untaken time.Duration) + }{ + {"defaults", nil, true, func(t *testing.T, cap int64, kept, step, untaken time.Duration) { + if cap != 16<<20 || kept != 168*time.Hour || step != 30*time.Minute || untaken != 24*time.Hour { + t.Errorf("defaults = %d %s %s %s", cap, kept, step, untaken) + } + }}, + {"set", map[string]string{envRunsCap: "8\n", envRunsKept: "48h", envStepBound: "2s", envUntakenBound: "1m"}, true, + func(t *testing.T, cap int64, kept, step, untaken time.Duration) { + if cap != 8<<20 || kept != 48*time.Hour || step != 2*time.Second || untaken != time.Minute { + t.Errorf("settings = %d %s %s %s", cap, kept, step, untaken) + } + }}, + {"a cap that is no whole number", map[string]string{envRunsCap: "8.5"}, false, nil}, + {"a time that is no duration", map[string]string{envStepBound: "30"}, false, nil}, + {"kept past its maximum", map[string]string{envRunsKept: "169h"}, false, nil}, + } + for _, r := range rows { + t.Run(r.name, func(t *testing.T) { + env := map[string]string{} + for k, v := range r.files { + env[k] = write(r.name+k, v) + } + s, err := runSettings(func(k string) string { return env[k] }) + if (err == nil) != r.ok { + t.Fatalf("runSettings = %+v, %v", s, err) + } + if r.check != nil { + r.check(t, s.CapBytes, s.Kept, s.StepBound, s.Untaken) + } + }) + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go index 27105fb6..5fd6c7dd 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs.go @@ -1,13 +1,10 @@ package main import ( - "context" "encoding/json" "errors" "fmt" - "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" @@ -41,7 +38,8 @@ func answerVerbs(conn *process.Conn, seats *seatlist.Part, v pipeline.Verbs) (fu return conn.AnswerSeat(seat.Seat, seatlist.Definition(seats.Reader, seat.Seat), writeHandlers(v)) } -// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, start, and ask-again. +// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, start, and ask-again; and +// the test runs' verbs test-runs and test-runs-drop (A-469g, A-470i). func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler { return map[string]process.SeatHandler{ seat.VerbRun: verbOf(func(c string, in seat.RunInput) (any, error) { return v.Run(c, in) }), @@ -51,24 +49,17 @@ func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler { seat.VerbCancel: verbOf(func(c string, in seat.CancelInput) (any, error) { return v.Cancel(c, in) }), seat.VerbStart: verbOf(func(c string, in seat.StartInput) (any, error) { return v.Start(c, in) }), seat.VerbAskAgain: verbOf(func(c string, in seat.AskAgainInput) (any, error) { return v.AskAgain(c, in) }), - } -} - -// dropBound deletes the bound of an ask a cancelled pipeline no longer waits on. -func dropBound(ctx context.Context, bounds jetstream.KeyValue) func(string) error { - return func(ask string) error { - if err := bounds.Delete(ctx, ask); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { - return err - } - return nil + // §728, A-469g: a test run's drop is answered by the process, the only writer of pipelines and test-runs. + seat.VerbTestRunsDrop: verbOf(func(c string, in seat.TestRunsDropInput) (any, error) { return v.TestRunsDrop(c, in) }), + // A-470i: test-runs too, since the process holds test-runs-cap-mib, the cap its fill is said against. + seat.VerbTestRuns: verbOf(func(_ string, in seat.TestRunsInput) (any, error) { return v.Runs.TestRuns(in) }), } } // writeVerbs are the verbs over the hearer and the starter, their hand acts recorded through the seat mesh-hand-acts. -func writeVerbs(ctx context.Context, h *pipeline.Hearer, s *pipeline.Starter, view func(string) (pipeline.Half, bool), - calls *process.Conn, bounds jetstream.KeyValue) pipeline.Verbs { +func writeVerbs(h *pipeline.Hearer, s *pipeline.Starter, view func(string) (pipeline.Half, bool), calls *process.Conn) pipeline.Verbs { candidate := func(group, branch string, revision int64) (pipeline.Half, error) { return readCandidate(calls.CallKept, catalog.CatalogRevisionEvent{Kind: "candidate", Group: group, Branch: branch, Revision: revision}) } - return pipeline.Verbs{Hearer: h, Starter: s, View: view, Candidate: candidate, Acts: meshhandactsclient.Recorder{Ask: calls.Call}, Drop: dropBound(ctx, bounds)} + return pipeline.Verbs{Hearer: h, Starter: s, View: view, Candidate: candidate, Acts: meshhandactsclient.Recorder{Ask: calls.Call}} } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go index 9e2d5d28..a8f84fcd 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/verbs_test.go @@ -23,7 +23,7 @@ func TestEveryWriteVerbIsServed(t *testing.T) { got = append(got, v) } slices.Sort(got) - want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun} + want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun, seat.VerbTestRuns, seat.VerbTestRunsDrop} slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("the process answers %v, want %v", got, want) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go index dac58bb4..3a620f52 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "sync" "sync/atomic" "time" @@ -12,6 +13,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/seatlist" "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-pipeline/internal/pipeline" @@ -26,6 +28,70 @@ type runnerViews struct { // Findings is the runner's held finding writer: it raises the finding pipeline-failed (§33) and says the runner's own // findings (A-206). Findings *findingsWriter + // Holders follows the seat-holders view of the seat mesh-module-assigner's claimant (§535, §647; A-466z). + Holders *holderView +} + +// holderView follows the view seat-holders of the module the catalog halves name as the claimant of the seat +// mesh-module-assigner (seatlist.HolderSeat), as the runner's manifest consumes it, `seat:mesh-module-assigner.views.seat-holders` +// (design 47 G §535, §647 as A-466z amends them; §717): a claimant named anew drops the view followed and follows the new one's. +// Nothing is asked on a timer: it is read again whenever a branch's half is held or advances. +type holderView struct { + reader *views.ProcessReader + catalog *pipeline.Views + logf func(string, ...any) + mu sync.Mutex + owner string + pic *views.Picture +} + +// refresh follows the claimant's seat-holders view when the claimant is another than the one followed. +func (h *holderView) refresh() { + var halves []pipeline.Half + for _, b := range h.catalog.Branches() { + if half, ok := h.catalog.View(b); ok { + halves = append(halves, half) + } + } + claimant, ok := pipeline.ClaimantOf(halves, seatlist.HolderSeat) + h.mu.Lock() + defer h.mu.Unlock() + if !ok || claimant == h.owner { + return + } + if h.owner != "" { + if err := h.reader.Drop(h.owner, seatHoldersState); err != nil { + h.logf("the seat-holders view of %s is not dropped: %v", h.owner, err) + } + h.owner, h.pic = "", nil + } + pic, err := h.reader.Follow(claimant, seatHoldersState, claimant+"."+seatHoldersState) + if err != nil { + h.logf("the seat-holders view of %s is not followed: %v", claimant, err) + return + } + h.owner, h.pic = claimant, pic + h.logf("the seat-holders view of %s, the claimant of %s, is followed", claimant, seatlist.HolderSeat) +} + +// seatHoldersState is the seat mesh-module-assigner's view of who holds each seat (§616, §699). +const seatHoldersState = "seat-holders" + +// Holders are the modules the followed view names for seatName, and whether its first view is read: before it, and with +// no claimant followed, nothing is judged (A-466y). +func (h *holderView) Holders(seatName string) ([]string, bool) { + h.mu.Lock() + pic := h.pic + h.mu.Unlock() + if pic == nil { + return nil, false + } + select { + case <-pic.Follower().Ready(): + default: + return nil, false + } + return seatlist.HoldersIn(func() (map[string]json.RawMessage, error) { return pic.Follower().Values(), nil })(seatName), true } // startViews wires the runner's views on hub and asks each first view; register every other handler on hub before or @@ -88,7 +154,7 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr return nil, err } reader.Start(ctx, js, cred, status) - return &runnerViews{Registrations: regs, Catalog: cv, Findings: findings}, nil + return &runnerViews{Registrations: regs, Catalog: cv, Findings: findings, Holders: &holderView{reader: reader, catalog: cv, logf: logf}}, nil } // catalogView is a branch's meshcatalogclient.Catalog as the runner reads it. diff --git a/modules/mesh-delivery-pipeline/go.mod b/modules/mesh-delivery-pipeline/go.mod index 50349fa3..58a8de3f 100644 --- a/modules/mesh-delivery-pipeline/go.mod +++ b/modules/mesh-delivery-pipeline/go.mod @@ -9,6 +9,8 @@ require ( ) require ( + git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go index 487cc8c5..bf9b7dda 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go @@ -1,6 +1,7 @@ package pipeline import ( + "errors" "fmt" "time" @@ -53,7 +54,12 @@ type Ager struct { Get(key string) (seat.KeptResult, bool, error) Delete(key string) error } - Now func() time.Time + // Runs is the state test-runs, RunOutputs deletes an object of test-runs-objects and RunsKept is the setting + // test-runs-kept: an ended run is deleted with its output object once RunsKept has passed since its end (§725, A-470a). + Runs TestRuns + RunOutputs func(name string) error + RunsKept time.Duration + Now func() time.Time // Logf is the runner's journal: each delete writes a line naming the key or object and its retention. Logf func(string, ...any) } @@ -69,7 +75,16 @@ func (a *Ager) Age() { if a == nil || a.Log == nil { return } - for _, r := range []Retention{Retention7, Retention30, Retention90} { + retentions := []Retention{Retention7, Retention30, Retention90} + if a.Runs != nil { + // The test runs age first, in the same pass, so the pipeline that names a run's key is still kept (A-470a). + kept := a.RunsKept + if kept <= 0 { + kept = DefaultRunSettings.Kept + } + retentions = append([]Retention{{RetentionTestRuns, kept}}, retentions...) + } + for _, r := range retentions { if err := a.ageAt(r); err != nil { a.logf("aged: the data past %s are not aged: %v", r.Name, err) } @@ -103,8 +118,57 @@ func (a *Ager) ageAt(r Retention) error { return a.Aged.Put(r.Name, moved) } +// RetentionTestRuns is the name, in the state aged, of the test runs' retention, the setting test-runs-kept (§725). +const RetentionTestRuns = "test-runs" + +// ageRun deletes the test run the ended step of e read, and its output object, once its kept time has passed since the +// step's end, in the step log's order (§725, A-470a): the run's key is read from e's pipeline, its step's Shared run. +func (a *Ager) ageRun(r Retention, e seat.LogEntry) { + if e.What != seat.LogEntryWhatEnded { + return + } + p, _, err := a.Pipelines.Get(e.Pipeline) + if err != nil { + return + } + run := "" + for _, s := range p.Steps { + if s.Step == e.Step && s.Artifact == e.Artifact { + run = sharedRun(s) + } + } + if run == "" { + return + } + for { + t, rev, err := a.Runs.Get(run) + // The entry is past the retention (ageAt), and a run ends no later than the reader's end it was read at: a run not + // ended yet is aged at a later reader's end. + if err != nil || t.Ended == nil { + return + } + if t.Output != "" && a.RunOutputs != nil { + if err := a.RunOutputs(t.Output); err != nil { + a.logf("aged: the output %s of the test run %s is not deleted, so the run is kept: %v", t.Output, run, err) + return + } + } + // Deleted at the revision read: a rerun that wrote the key again since is read again, and a run it started is kept. + if err := a.Runs.Delete(run, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + a.logf("aged: the test run %s is not deleted: %v", run, err) + return + } + a.logf("aged: deleted the test run %s and its output (%s after its end)", run, r.Age) + return + } +} + func (a *Ager) ageEntry(r Retention, e seat.LogEntry, now time.Time) { switch r.Name { + case RetentionTestRuns: + a.ageRun(r, e) case Retention7.Name: if e.Output != "" && a.Outputs != nil { if err := a.Outputs(e.Output); err != nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/approval.go b/modules/mesh-delivery-pipeline/internal/pipeline/approval.go index beeff984..c3d32346 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/approval.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/approval.go @@ -38,7 +38,7 @@ func approvalDue(p seat.Pipeline) int { } continue } - if s.Outcome == seat.StepResultOutcomeWaiting || s.Outcome == seat.StepResultOutcomeRunning { + if s.Outcome == seat.StepResultOutcomeWaiting || open(s) { return -1 } if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go b/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go index bfb3063e..f3847b25 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/bounds.go @@ -1,20 +1,19 @@ package pipeline import ( - "errors" "fmt" "strings" "time" - seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage" nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode" ) -// AskBound is how long an ask on a build seat's work queue may wait for its holder to take it (design 47 G §535, §647): five -// minutes, the bound of a key of the state `bounds` with its own time to live (A-77, A-104), never a timer. -const AskBound = 5 * time.Minute +// DropBound is how long a superseded step's drop may go unanswered before it is said (design 47 G A-222, A-77): five minutes, +// a key of the state `bounds` with its own time to live. An ask waiting for its holder has no such bound (§535, §647 as +// A-466y amends them): it is timed by §86's build-capacity finding and, for a test run, by run-untaken-bound (§725). +const DropBound = 5 * time.Minute // TakenSeat is a build seat whose holder says `ask-taken` when it takes an ask (design 47 G §602): where the runner hears it. type TakenSeat struct { @@ -23,15 +22,15 @@ type TakenSeat struct { Subject string } -// TakenSeats are the three build seats, from their contracts: the seat mesh-builds and the seat mesh-wants say no ask-taken, -// so no bound is armed for their asks. +// TakenSeats are the three build seats, from their contracts: the seat mesh-builds and the seat mesh-wants say no ask-taken. var TakenSeats = []TakenSeat{ {Seat: nodebuildgo.Seat, Subject: nodebuildgo.SubjectAskTakenAll}, {Seat: nodebuildnode.Seat, Subject: nodebuildnode.SubjectAskTakenAll}, {Seat: nodebuildimage.Seat, Subject: nodebuildimage.SubjectAskTakenAll}, } -// BoundSeat says whether the holder of a seat says ask-taken, so an ask put on its work queue has a bound. +// BoundSeat says whether the holder of a seat says ask-taken: a build seat, whose asks are dropped and whose test steps share +// a run (§724). func BoundSeat(name string) bool { for _, s := range TakenSeats { if s.Seat == name { @@ -41,63 +40,6 @@ func BoundSeat(name string) bool { return false } -// Arm puts a bound of one ask's wait for its holder: a key of the state `bounds` named for the ask, written with its own -// time to live. It is armed before the ask is put, so an ask-taken heard at once finds it. -type Arm func(ask string) error - -// NotTaken ends the step asked under ask that no holder took within its bound, the bound's expiry heard (design 47 G §32, -// §535, §647, A-77): the step ends `could not run` naming the seat, so its pipeline ends failed once the others end, never -// a silent wait. A step that ended or was never asked of a seat takes nothing. -func (h *Hearer) NotTaken(ask string, bound time.Duration) error { - pkeys, err := h.Store.Keys("*.*.*") - if err != nil { - return fmt.Errorf("the pipelines are not listed: %w", err) - } - for _, key := range pkeys { - p, _, err := h.Store.Get(key) - if err != nil || p.Outcome != seat.PipelineOutcomeRunning || stepOfAsk(p, ask) < 0 { - continue - } - return h.endNotTaken(key, ask, bound) - } - return nil -} - -func (h *Hearer) endNotTaken(key, ask string, bound time.Duration) error { - for { - p, rev, err := h.Store.Get(key) - if err != nil { - return fmt.Errorf("the pipeline %s is not read: %w", key, err) - } - i := stepOfAsk(p, ask) - if i < 0 || p.Steps[i].Outcome != seat.StepResultOutcomeRunning { - return nil - } - now := h.now() - s := &p.Steps[i] - s.Outcome, s.Ended = seat.StepResultOutcomeCouldNotRun, &now - s.Why = fmt.Sprintf("the seat %s has no holder that took ask %s within %s", s.Seat, ask, bound) - p.Unlogged = append(p.Unlogged, endEntry(key, *s, now)) - ended := endPipeline(&p, now) - if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) { - continue - } else if err != nil { - return fmt.Errorf("pipeline %s cannot be written: %w", key, err) - } - h.Failures.StepEnded(h.Store, p, *s) - if err := h.say(seat.EventStepEnded, p, s); err != nil { - return fmt.Errorf("pipeline %s's %s is written but not said: %w", key, seat.EventStepEnded, err) - } - h.appendUnlogged(key) - h.Ager.Age() - h.Fills.Check() - return h.said(p, false, ended) - } -} - -// IsAskKey says whether a key of the state `bounds` is an ask's bound. -func IsAskKey(key string) bool { return strings.HasPrefix(key, "step-") && !strings.Contains(key, ".") } - // DropBoundPrefix begins the key of a superseded step's drop in the state `bounds`, `drop..` (design 47 G // A-222, A-77): its five-minute expiry says the drop unanswered. const DropBoundPrefix = "drop." diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go index bee58513..a179ebd5 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go @@ -75,7 +75,7 @@ func TestAPipelinePinsItsCatalogRevisionWhileItRuns(t *testing.T) { } want(0, delivery.SubjectFreed, "pipeline."+pkey, 12) said = nil - if n, err := h.Supersede(seat.Pipeline{Pipeline: "main.6.app", Branch: "main", Source: 6, Module: "app", Commit: "c2", Fingerprint: "src2:bb", Outcome: seat.PipelineOutcomeRunning}, nil, nil); n != 1 || err != nil { + if n, err := h.Supersede(seat.Pipeline{Pipeline: "main.6.app", Branch: "main", Source: 6, Module: "app", Commit: "c2", Fingerprint: "src2:bb", Outcome: seat.PipelineOutcomeRunning}, nil); n != 1 || err != nil { t.Fatalf("supersede %d (%v)", n, err) } want(0, delivery.SubjectFreed, "pipeline."+pkey, 12) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index 653ca3a3..a9bec3c3 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -77,7 +77,18 @@ func startOf(p seat.Pipeline, now time.Time) time.Time { // could not run naming the missing seat; the module's steps wait. It returns the steps and the asks to put on the seats' // work queues, one per running step. func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string) ([]seat.StepResult, []Asked) { + return place(p, reg, entries, now, asker, true) +} + +// place is Place; share false places each build step its own ask, as a step asked before the test runs is (§730, A-468k). +// With share, the build steps of one candidate, level and share digest are asked once (§724): the first artifact's step, in +// reference order, asks the run and names it in Shared; every other step of the key reads it, keeping no ask of its own and +// naming the run's ask and the step that asked it (A-469m, A-469q). +func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string, share bool) ([]seat.StepResult, []Asked) { entry := entries[p.Module] + candidate, named := CandidateOf(p) + share = share && named + askOfRun := map[string]int{} // a test-runs key to the index of its ask in asks arts := append([]catalog.Artifact(nil), entry.Artifacts...) sort.Slice(arts, func(i, j int) bool { return arts[i].Ref < arts[j].Ref }) var steps []seat.StepResult @@ -118,8 +129,27 @@ func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog } s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now s.Ask = AskID(startOf(p, now), p.Pipeline, a.Ref, step) + body := stepAsk(p, entry, a, step, s.Ask, asker, now) + ask := Asked{Seat: bs, ID: s.Ask, Ask: body} + if share && BoundSeat(bs) { + digest := ShareDigest(p.Fingerprint, step, status, body.Step.Fetch) + run := TestRunKey(candidate, step, digest) + reader := seat.TestRunReader{Pipeline: p.Pipeline, Artifact: a.Ref} + if j, asked := askOfRun[run]; asked { + // Another artifact's step of this pipeline asks the run: this one reads it. + first := &asks[j] + first.Readers = append(first.Readers, reader) + s.Ask, s.Outcome, s.On = "", seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode + s.Shared = []seat.Shared{{Run: run, Ask: first.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: first.Readers[0].Artifact}}} + steps = append(steps, s) + continue + } + // A test run's steps wait for a build node from its ask until the holder's ask-taken (A-469b). + s.Shared, s.Outcome, s.On = []seat.Shared{{Run: run, Ask: s.Ask}}, seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode + ask.Run, ask.Level, ask.Digest, ask.Readers = run, step, digest, []seat.TestRunReader{reader} + askOfRun[run] = len(asks) + } steps = append(steps, s) - ask := Asked{Seat: bs, ID: s.Ask, Ask: stepAsk(p, entry, a, step, s.Ask, asker, now)} if a.Package != nil { ask.Package = a.Package.Name } @@ -156,6 +186,10 @@ type Asked struct { // Package is the name of the artifact's package, from the catalog entry, for a step on a build seat; its kept results // are the ask's (§25, §416). Package string + // Run is the test run the ask asks, its key in test-runs, with its level, its share digest and the steps of the pipeline + // reading it, the asking step first (§724, §725); empty for an ask of no test run. + Run, Level, Digest string + Readers []seat.TestRunReader } func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step, id, asker string, now time.Time) nodebuildgo.StepAsk { @@ -170,8 +204,9 @@ func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step, Module: p.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}} } -// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. -func askAll(asker StepAsker, arm Arm, key string, asks []Asked, recovery bool) error { +// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its +// holder with no bound of the runner's own (§535, §647 as A-466y amends them). +func askAll(asker StepAsker, key string, asks []Asked, recovery bool) error { var first error for _, a := range asks { accept, ok := acceptStep[a.Seat] @@ -179,15 +214,6 @@ func askAll(asker StepAsker, arm Arm, key string, asks []Asked, recovery bool) e accept = to } err := fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat) - if ok { - // A build seat's holder says ask-taken: the ask's bound is armed before it is put (§535, §647). - if arm != nil && BoundSeat(a.Seat) { - if aerr := arm(a.ID); aerr != nil { - err = fmt.Errorf("its bound is not armed: %w", aerr) - ok = false - } - } - } if ok { err = asker.AskStep(accept, a.ID, a.Ask) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index 5336fef0..eec07992 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -328,3 +328,207 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) { t.Errorf("asked %d, want the build, the image's three steps and the composition", len(asks)) } } + +// testRunsMem is the state test-runs in memory, a write by compare-and-set on the revision read. +type testRunsMem map[string]struct { + r Run + rev uint64 +} + +func (m testRunsMem) Create(key string, r Run) error { + if _, ok := m[key]; ok { + return ErrExists + } + m[key] = struct { + r Run + rev uint64 + }{r, 1} + return nil +} + +func (m testRunsMem) Get(key string) (Run, uint64, error) { + e, ok := m[key] + if !ok { + return Run{}, 0, ErrNoRun + } + e.r.Readers = append([]RunReader(nil), e.r.Readers...) + return e.r, e.rev, nil +} + +func (m testRunsMem) Delete(key string, rev uint64) error { + if e, ok := m[key]; ok && e.rev != rev { + return ErrChanged + } + delete(m, key) + return nil +} + +func (m testRunsMem) Update(key string, r Run, rev uint64) (uint64, error) { + if m[key].rev != rev { + return 0, ErrChanged + } + m[key] = struct { + r Run + rev uint64 + }{r, rev + 1} + return rev + 1, nil +} + +// One test run per candidate, level and tested packages (design 47 G §724, A-469c): a module whose program and tools bundle +// test one directory asks one lint, one unit and one integration step, the other artifact's steps reading them; a second +// pipeline of the same candidate reads the runs; a run ended could not run is asked again; a second revision with the same +// digests asks its own runs. +func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) { + reg := catalog.Registration{Module: "app", Branches: []string{"main"}} + entries := map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, + } + store, runs, asks := memStore{}, testRunsMem{}, &stepAsks{} + s := &Starter{Pipelines: memCreated{store}, Store: store, TestRuns: runs, Say: func(string, any) error { return nil }, Steps: asks, + Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }} + start := func(source, revision int64) seat.Pipeline { + t.Helper() + if ok, err := s.Start(Begin{Branch: "main", Source: source, Module: "app", Commit: "c1", Fp: "src2:aa", Revision: revision, Entries: entries}); !ok || err != nil { + t.Fatalf("start %d = %v, %v", source, ok, err) + } + p, _, _ := store.Get(Key("main", source, "app")) + return p + } + testAsks := func(from int) []string { + var out []string + for _, a := range asks.asks[from:] { + if b, ok := a.(nodebuildgo.StepAsk); ok { + out = append(out, b.Step.Artifact+" "+b.Step.Step) + } + } + return out + } + p := start(4, 9) + if got := strings.Join(testAsks(0), "|"); got != "app:program lint|app:program unit|app:program integration" { + t.Fatalf("the test steps asked = %q, want the program's three alone", got) + } + for _, level := range buildSteps { + asker, reader := stepNamed(p, "app:program", level), stepNamed(p, "app:tools", level) + if asker.Ask == "" || len(asker.Shared) != 1 || asker.Shared[0].Ask != asker.Ask || asker.Shared[0].AskedBy != nil { + t.Errorf("the asking %s step = %+v, want its ask named in shared with no asked-by", level, asker) + } + if reader.Ask != "" || reader.Outcome != seat.StepResultOutcomeWaitingForABuildNode || reader.On != seat.WaitOnBuildNode || len(reader.Shared) != 1 || reader.Shared[0] != (seat.Shared{ + Run: asker.Shared[0].Run, Ask: asker.Ask, AskedBy: reader.Shared[0].AskedBy}) || reader.Shared[0].AskedBy == nil || + *reader.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) { + t.Errorf("the reading %s step = %+v, want no ask and the program's run named", level, reader) + } + r, _, err := runs.Get(asker.Shared[0].Run) + if err != nil || r.Ask != asker.Ask || r.Level != level || len(r.Readers) != 2 || !strings.HasPrefix(asker.Shared[0].Run, "main.9."+level+".") { + t.Errorf("the run of %s = %+v, %v: want the program's ask and both artifacts reading it", level, r, err) + } + } + // A second pipeline of the same candidate (another source revision of the same catalog revision) reads the runs. + n := len(asks.asks) + q := start(5, 9) + if got := testAsks(n); len(got) != 0 { + t.Errorf("the second pipeline of the candidate asked %q, want it to read the runs", got) + } + unit := stepNamed(q, "app:program", "unit") + if unit.Ask != "" || unit.Shared[0].Ask != stepNamed(p, "app:program", "unit").Ask || *unit.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) { + t.Errorf("the second pipeline's unit step = %+v, want it reading the first's run", unit) + } + if r, _, _ := runs.Get(unit.Shared[0].Run); len(r.Readers) != 4 { + t.Errorf("the unit run's readers = %+v, want both pipelines' two artifacts", r.Readers) + } + // A run ended could not run is never read: the next step of its key asks it again. + key := stepNamed(p, "app:program", "lint").Shared[0].Run + r, rev, _ := runs.Get(key) + ended := time.Unix(9, 0) + r.Ended, r.Outcome = &ended, seat.StepResultOutcomeCouldNotRun + _, _ = runs.Update(key, r, rev) + n = len(asks.asks) + start(6, 9) + if got := strings.Join(testAsks(n), "|"); got != "app:program lint" { + t.Errorf("after a lint run that could not run, asked %q, want the lint step alone", got) + } + // A second revision with the same digests asks its own runs, so a failed test runs again. + n = len(asks.asks) + start(7, 10) + if got := len(testAsks(n)); got != 3 { + t.Errorf("a second revision asked %d test steps, want its own three", got) + } +} + +// A seat with no holder is read only from the seat-holders view (design 47 G §535, §647 as A-466y and A-466z amend them): +// once that view's first view is read, a build step whose seat it names no holder for ends could not run naming the seat, +// and the pipeline ends failed at once naming it, every other step `not asked`, each said, none asked (A-470g, A-470h); +// before it is read, or with a holder named, the step is asked and waits with no bound of the runner's own; the claimant of +// the view is the one module the catalog halves name. +func TestASeatWithNoHolderIsReadFromTheSeatHoldersView(t *testing.T) { + reg, entries := placeFixture() + for _, c := range []struct { + name string + holders []string + read bool + asked bool + }{{"the view not read yet", nil, false, true}, {"a holder named", []string{"mesh-build-go"}, true, true}, {"no holder named", nil, true, false}} { + t.Run(c.name, func(t *testing.T) { + store, asks := memStore{}, &stepAsks{} + var said []int + s := &Starter{Pipelines: memCreated{store}, Say: func(string, any) error { return nil }, Steps: asks, Name: "the-runner", + Registration: func(string) (catalog.Registration, bool) { return reg, true }, + Holders: func(string) ([]string, bool) { return c.holders, c.read }, + Ended: func(_ seat.Pipeline, steps []int) error { said = steps; return nil }} + if ok, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Entries: entries}); !ok || err != nil { + t.Fatalf("start = %v, %v", ok, err) + } + p, _, _ := store.Get("main.4.app") + lint := stepNamed(p, "app:program", "lint") + asked := false + for _, id := range asks.ids { + asked = asked || id == lint.Ask + } + if c.asked && (lint.Outcome != seat.StepResultOutcomeRunning || !asked) { + t.Errorf("the lint step = %+v, asked %v; want it running and asked", lint, asked) + } + if c.asked { + if p.Outcome != seat.PipelineOutcomeRunning || said != nil { + t.Errorf("the pipeline = %s (%s), ended steps said %v; want it running", p.Outcome, p.Why, said) + } + return + } + if lint.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(lint.Why, "node-build-go has no holder") || len(asks.ids) != 0 { + t.Errorf("the lint step = %+v, %d asked; want could not run naming the seat, nothing asked", lint, len(asks.ids)) + } + if want := "the seat node-build-go has no holder: the seat-holders view names none"; p.Outcome != seat.PipelineOutcomeFailed || p.Why != want || p.Ended == nil || p.EndCount != 1 { + t.Errorf("the pipeline = %s (%q), ended %v, count %d; want failed at once, %q", p.Outcome, p.Why, p.Ended, p.EndCount, want) + } + ended := 0 + for _, st := range p.Steps { + switch { + case st.Seat == "node-build-go", st.Outcome == seat.StepResultOutcomeNotAsked: + ended++ + case st.Outcome == seat.StepResultOutcomeBypassed, st.Outcome == seat.StepResultOutcomeCouldNotRun && st.Seat == "": + // ended by Place itself: a bypass, or a language no build seat names + default: + t.Errorf("the step %s %s = %s; want not asked", st.Artifact, st.Step, st.Outcome) + } + } + if ended < 3 || len(said) != ended { + t.Errorf("%d steps ended at placing, %d said; want each said", ended, len(said)) + } + }) + } + claims := func(seats ...string) map[string]any { + var cs []any + for _, n := range seats { + cs = append(cs, map[string]any{"name": n, "scope": "mesh"}) + } + return map[string]any{"module": "x", "version": "0.1", "claims": cs} + } + half := Half{Entries: map[string]catalog.CatalogEntry{"assigner": {Module: "assigner", Manifest: claims("mesh-module-assigner", "mesh-wants")}, + "app": {Module: "app", Manifest: claims()}}} + if got, ok := ClaimantOf([]Half{half, {}}, "mesh-module-assigner"); !ok || got != "assigner" { + t.Errorf("the claimant = %q, %v", got, ok) + } + other := Half{Entries: map[string]catalog.CatalogEntry{"other": {Module: "other", Manifest: claims("mesh-module-assigner")}}} + if got, ok := ClaimantOf([]Half{half, other}, "mesh-module-assigner"); ok { + t.Errorf("halves that disagree name %q", got) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read.go b/modules/mesh-delivery-pipeline/internal/pipeline/read.go index b244afbb..9fed9e40 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read.go @@ -95,12 +95,12 @@ func (r Reads) Stands() (seat.StandsAnswer, error) { if s.Ended != nil || s.Started == nil { continue } - if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting { + if !open(s) && s.Outcome != seat.StepResultOutcomeWaiting { continue } out = append(out, seat.Wait{Pipeline: p.Pipeline, Module: p.Module, Branch: p.Branch, Commit: p.Commit, Group: p.Group, Run: p.Run, Source: p.Source, Revision: p.Revision, Fingerprint: p.Fingerprint, Step: s.Step, - Artifact: s.Artifact, On: s.On, Since: *s.Started, V: 1}) + Artifact: s.Artifact, On: s.On, Since: *s.Started, V: 1, Ask: askOfWait(s), Shared: sharedOfWait(s)}) } } sort.SliceStable(out, func(i, j int) bool { return out[i].Since.Before(out[j].Since) }) @@ -219,3 +219,19 @@ func startedOf(p seat.Pipeline) time.Time { } return *p.Started } + +// askOfWait is a waiting step's own ask (design 47 G §731, §496): a step reading another's test run keeps none (A-469m). +func askOfWait(s seat.StepResult) string { return s.Ask } + +// sharedOfWait is the run a waiting step waits on, where the runner knows it (§731; A-469r, A-469v): a test step's run key +// and ask, as one entry with no machine and no asker. +func sharedOfWait(s seat.StepResult) []seat.Shared { + if len(s.Shared) == 0 { + return nil + } + out := make([]seat.Shared, 0, len(s.Shared)) + for _, x := range s.Shared { + out = append(out, seat.Shared{Run: x.Run, Ask: x.Ask, Machine: x.Machine}) + } + return out +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go index 20c544ab..8dad23b1 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" ) @@ -150,3 +151,55 @@ func TestShowAndKeptCarryTheStoresFill(t *testing.T) { t.Errorf("show with the fill unread: %v", err) } } + +// A step that reads a test run says its own step-started at every place it is asked, naming the run's key and ask in shared +// and no asker (design 47 G §726, §729; A-469a, A-469q), and stands names each waiting step's own ask and, for a test step, +// the run it waits on (§731; A-469r, A-469v). +func TestStandsAndStepStartedNameWhatAReadingStepWaitsOn(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + asker, reader := stepNamed(p, "app:program", "unit"), stepNamed(p, "app:tools", "unit") + started := map[string]seat.StepResult{} + for _, e := range f.said { + started[e.Result.Artifact+" "+e.Result.Step] = e.Result + } + for _, s := range []seat.StepResult{asker, reader} { + got, ok := started[s.Artifact+" unit"] + if !ok || len(got.Shared) != 1 || got.Shared[0] != (seat.Shared{Run: asker.Shared[0].Run, Ask: asker.Ask}) { + t.Errorf("the step-started of %s = %+v (said %v), want the run's key and ask, no asker", s.Artifact, got, ok) + } + } + if _, ok := started[" composition"]; !ok { + t.Error("the composition step's step-started is not said") + } + pipes := memKept{} + put(t, pipes, p.Pipeline, f.pipeline(p)) + a, err := Reads{Pipelines: pipes, Kept: memKept{}}.Stands() + if err != nil { + t.Fatal(err) + } + waits := map[string]seat.Wait{} + for _, w := range a.Stands { + waits[w.Artifact+" "+w.Step] = w + } + want := []seat.Shared{{Run: asker.Shared[0].Run, Ask: asker.Ask}} + if w := waits["app:program unit"]; w.Ask != asker.Ask || len(w.Shared) != 1 || w.Shared[0] != want[0] { + t.Errorf("the asking step stands as %+v, want its ask and its run", w) + } + if w := waits["app:tools unit"]; w.Ask != "" || len(w.Shared) != 1 || w.Shared[0] != want[0] { + t.Errorf("the reading step stands as %+v, want no ask of its own and the run's key and ask", w) + } + if w := waits[" composition"]; w.Ask == "" || len(w.Shared) != 0 { + t.Errorf("the composition step stands as %+v, want its own ask and no run", w) + } + // Resume asks the steps again and says their step-started again (A-469a). + n := len(f.said) + if _, err := f.s.Resume(f.store, "main", Half{Entries: map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}); err != nil { + t.Fatal(err) + } + if len(f.said) == n { + t.Error("resume said no step-started") + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go index a2a2db7b..0016e988 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go @@ -2,12 +2,14 @@ package pipeline import ( "context" + "encoding/json" "fmt" "sort" "sync" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/catalogue" ) // Status is a step's status for one artifact of a module, read from the module's registration and never from a @@ -212,3 +214,35 @@ func (r *Readiness) Observe(rev int64) (first, advanced bool) { } return false, false } + +// ClaimantOf is the module the catalog halves name as the claimant of seatName (design 47 G §614, A-82): the one module whose +// entry's manifest claims it, the same in every half that names one; none, more than one, or halves that disagree name none. +func ClaimantOf(halves []Half, seatName string) (string, bool) { + found := "" + for _, h := range halves { + var named []string + for _, e := range h.Entries { + raw, err := json.Marshal(e.Manifest) + if err != nil { + continue + } + m, err := catalogue.Decode(raw) + if err != nil { + continue + } + for _, c := range m.Claims { + if c.Name == seatName { + named = append(named, e.Module) + } + } + } + switch { + case len(named) == 0: + continue + case len(named) > 1, found != "" && found != named[0]: + return "", false + } + found = named[0] + } + return found, found != "" +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/run.go b/modules/mesh-delivery-pipeline/internal/pipeline/run.go index 16e85b51..5063e81c 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/run.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/run.go @@ -229,9 +229,11 @@ func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (b } p.Fingerprint, p.Repository, p.Revision = e.Fp, e.Repository, revision var asks []Asked + var ended []int if s.Registration != nil { if reg, ok := s.Registration(p.Module); ok { p.Steps, asks = Place(p, reg, half.Entries, now, s.Name) + asks, ended = s.holderless(&p, asks, now) } else { s.logf("%s has no registration yet: its pipeline is placed with no step", key) } @@ -246,9 +248,11 @@ func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (b if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, p.Module, p.Branch, p.Commit), ev); err != nil { s.logf("pipeline-started of %s is not said: %v", key, err) } + if len(ended) > 0 { + return true, s.endedAtPlacing(p, ended) + } if s.Steps != nil && len(asks) > 0 { - s.carry(key, asks) - if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil { + if err := s.askSteps(key, p.Commit, asks); err != nil { return true, err } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/run_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/run_test.go index 6dbc0dc1..d2670bc5 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/run_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/run_test.go @@ -79,7 +79,13 @@ func TestRunOnRequest(t *testing.T) { if _, _, err := store.Get(RunKey("fix/1", a.Run, "same")); err == nil { t.Error("an entry the members do not change got a pipeline") } - if len(said) != 2 || !strings.HasPrefix(said[0], seat.EventPipelineStarted+".") || len(asks.ids) == 0 { + // Each pipeline placed says pipeline-started, then step-started for each step it asks (A-469a). + started, steps := 0, 0 + for _, x := range said { + started += strings.Count(x, seat.EventPipelineStarted+".") + steps += strings.Count(x, seat.EventStepStarted+".") + } + if started != 2 || !strings.HasPrefix(said[0], seat.EventPipelineStarted+".") || len(asks.ids) == 0 || steps < len(asks.ids) { t.Errorf("placing said %v and asked %d steps", said, len(asks.ids)) } if ids, _ := s.UnplacedRuns(); len(ids) != 0 { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go new file mode 100644 index 00000000..e686f14f --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go @@ -0,0 +1,488 @@ +package pipeline + +import ( + "errors" + "fmt" + "strings" + "time" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// RunSettings are the runner's settings of its test runs (design 47 G §725; A-469d, A-469w, A-469x): the cap of test-runs +// the process judges a write by, how long an ended run is kept, the bound of a taken run's owed progress and the bound of +// an ask never taken. +type RunSettings struct { + CapBytes int64 + Kept time.Duration + StepBound time.Duration + Untaken time.Duration +} + +// The settings' defaults (§725; A-466n, A-466s): 16 MiB, 7 days, 30 minutes, 24 hours. +var DefaultRunSettings = RunSettings{CapBytes: 16 << 20, Kept: 7 * 24 * time.Hour, StepBound: 30 * time.Minute, Untaken: 24 * time.Hour} + +// The two bound keys of a test run in test-runs, each with its own time to live (§725; A-468b, A-470): `bound.`, +// written at the build seat's ask-taken of the run's ask and again at each step-progress of it, and `untaken.`, +// written at the run's start. +const ( + BoundPrefix = "bound." + UntakenPrefix = "untaken." +) + +// RunObjects is the state test-runs' own object store, test-runs-objects (A-469e): a run's output, kept while its key lives. +type RunObjects interface { + Put(name string, data []byte) error + // Get reads an object; ErrNoRun when it is gone. + Get(name string) ([]byte, error) + Delete(name string) error +} + +// RunClock is the bound keys of the test runs, keys of test-runs with their own time to live, each holding the ask of the +// run it bounds (A-470n). +type RunClock interface { + // Bound writes key again holding ask, with ttl as its time to live, by compare-and-set on the key's latest revision + // (retried when another write came between); it answers the revision written. + Bound(key, ask string, ttl time.Duration) (uint64, error) + // Bounded reads key: the ask it holds and its revision; ok false when it is not written. + Bounded(key string) (ask string, revision uint64, ok bool, err error) + // Unbound deletes key; a key gone is no fault. + Unbound(key string) error + // UnboundAt deletes key at revision; a later write of it (another run's bound) is kept, and a key gone is no fault. + UnboundAt(key string, revision uint64) error + // Has says whether key is written. + Has(key string) (bool, error) + // Keys are test-runs' keys. + Keys() ([]string, error) +} + +// ParseRunBound reads a key of test-runs as a run's bound: the run's key and whether it is the untaken bound. +func ParseRunBound(key string) (run string, untaken, ok bool) { + if run, ok = strings.CutPrefix(key, BoundPrefix); ok { + return run, false, run != "" + } + if run, ok = strings.CutPrefix(key, UntakenPrefix); ok { + return run, true, run != "" + } + return "", false, false +} + +// IsRunBound says whether a key of test-runs is a bound key rather than a run. +func IsRunBound(key string) bool { + _, _, ok := ParseRunBound(key) + return ok +} + +// runOfAsk is the test run asked under ask: the run its asking step names in Shared, read from the state pipelines. +func (h *Hearer) runOfAsk(ask string) (string, bool) { + if h.TestRuns == nil || ask == "" { + return "", false + } + pkeys, err := h.Store.Keys("*.*.*") + if err != nil { + return "", false + } + for _, key := range pkeys { + p, _, err := h.Store.Get(key) + if err != nil { + continue + } + if i := stepOfAsk(p, ask); i >= 0 { + run := sharedRun(p.Steps[i]) + return run, run != "" + } + } + return "", false +} + +// RunTaken takes the ask-taken of a test run's ask (§725, A-470): the node that took it is kept on the run, its untaken +// bound deleted and its clock begun, `bound.` written with test-step-bound as its time to live. An ask of no test +// run takes nothing. +func (h *Hearer) RunTaken(ask, node string) error { + run, ok := h.runOfAsk(ask) + if !ok { + return nil + } + for { + r, rev, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) { + return nil + } + if err != nil { + return fmt.Errorf("the test run %s is not read: %w", run, err) + } + if r.Ask != ask || r.Ended != nil { + h.logf("the ask-taken of ask %s is late: the test run %s names ask %s (ended %v), so it moves nothing (A-470n)", ask, run, r.Ask, r.Ended != nil) + return nil + } + if r.Node != node { + r.Node = node + if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return fmt.Errorf("the test run %s is not written: %w", run, err) + } + } + h.runStarted(run, r) + break + } + if err := armBound(h.Clock, h.TestRuns, BoundPrefix+run, run, ask, h.runSettings().StepBound); err != nil { + return fmt.Errorf("the clock of the test run %s is not begun: %w", run, err) + } + // The untaken bound is deleted only while it holds this ask: a rerun's own untaken bound is kept. + held, rev, ok, err := h.Clock.Bounded(UntakenPrefix + run) + if err != nil || !ok || held != ask { + return err + } + return h.Clock.UnboundAt(UntakenPrefix+run, rev) +} + +// armBound writes the bound key of the run under ask holding that ask, judged from a read of the run, then reads the run +// again (A-470n): when it no longer names ask, or has ended, the bound just written is deleted at its revision, so no +// bound written from the earlier read ends the run after it, and a bound another run wrote since is kept. +func armBound(clock RunClock, runs TestRuns, key, run, ask string, ttl time.Duration) error { + rev, err := clock.Bound(key, ask, ttl) + if err != nil { + return err + } + r, _, err := runs.Get(run) + if err != nil && !errors.Is(err, ErrNoRun) { + return err + } + if errors.Is(err, ErrNoRun) || r.Ended != nil || r.Ask != ask { + return clock.UnboundAt(key, rev) + } + return nil +} + +// RunProgress takes a build seat's step-progress of a test run's ask (§725, A-466p): its clock begins again, so a run that +// keeps saying its progress is never ended by it. An ask of no running, taken test run takes nothing. +func (h *Hearer) RunProgress(ask string) error { + run, ok := h.runOfAsk(ask) + if !ok { + return nil + } + r, _, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) || (err == nil && (r.Ask != ask || r.Ended != nil)) { + h.logf("the step-progress of ask %s is late: the test run %s names ask %q (ended %v), so it moves nothing (A-470n)", ask, run, r.Ask, r.Ended != nil) + return nil + } + if err != nil { + return fmt.Errorf("the test run %s is not read: %w", run, err) + } + return armBound(h.Clock, h.TestRuns, BoundPrefix+run, run, ask, h.runSettings().StepBound) +} + +// RunExpired takes the expiry of a test run's bound key (§725, §720 *When the clock ends*, A-466s): the run ends `could not +// run`, naming its ask and the bound, its ask dropped and every reader ended with it. An untaken bound of a run whose ask +// was taken, and a bound of a run ended already, end nothing. +func (h *Hearer) RunExpired(key string) error { + run, untaken, ok := ParseRunBound(key) + if !ok || h.TestRuns == nil { + return nil + } + r, _, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) { + return nil + } + if err != nil { + return fmt.Errorf("the test run %s is not read: %w", run, err) + } + // An untaken bound of a taken run, and a clock of a run not taken (a clock left from an ask before a rerun's), end nothing. + if r.Ended != nil || (untaken && r.Node != "") || (!untaken && r.Node == "") { + return nil + } + // The marker of an expiry holds no value (A-470o): the run ends only while its bound is still absent when read again, + // so an expiry heard after the bound was armed again for the run's own ask ends nothing. + if _, _, armed, err := h.Clock.Bounded(key); err != nil { + return fmt.Errorf("the bound %s is not read again: %w", key, err) + } else if armed { + h.logf("the expiry of %s ends nothing: the bound is armed again (A-470o)", key) + return nil + } + s := h.runSettings() + why := fmt.Sprintf("the ask %s of %s was taken by %s and said no progress within %s (test-step-bound)", r.Ask, r.Seat, r.Node, s.StepBound) + if untaken { + why = fmt.Sprintf("the ask %s of %s was not taken within %s of the run's start (run-untaken-bound)", r.Ask, r.Seat, s.Untaken) + } + if h.DropRun != nil { + if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil { + h.logf("the drop of ask %s of the test run %s is not answered: %v", r.Ask, run, err) + } + } + return h.EndRun(run, seat.TestRun{Ask: r.Ask, Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil) +} + +// TakeUp takes up again, at the runner's start, each test run with no end (§725): a taken run whose clock is gone, and an +// untaken run whose untaken bound is gone, passed its bound while the runner was away and ends as its expiry would. It +// says how many it ended. +func (h *Hearer) TakeUp() (int, error) { + if h.TestRuns == nil || h.Clock == nil { + return 0, nil + } + all, err := h.Clock.Keys() + if err != nil { + return 0, fmt.Errorf("the test runs are not listed: %w", err) + } + ended := 0 + var first error + for _, run := range all { + if IsRunBound(run) { + continue + } + r, _, err := h.TestRuns.Get(run) + if err != nil || r.Ended != nil { + continue + } + bound := UntakenPrefix + run + if r.Node != "" { + bound = BoundPrefix + run + } + has, err := h.Clock.Has(bound) + if err != nil || has { + if err != nil && first == nil { + first = err + } + continue + } + if err := h.RunExpired(bound); err != nil && first == nil { + first = err + } + ended++ + } + return ended, first +} + +func (h *Hearer) runSettings() RunSettings { + s := h.Runs + d := DefaultRunSettings + if s.CapBytes <= 0 { + s.CapBytes = d.CapBytes + } + if s.Kept <= 0 { + s.Kept = d.Kept + } + if s.StepBound <= 0 { + s.StepBound = d.StepBound + } + if s.Untaken <= 0 { + s.Untaken = d.Untaken + } + return s +} + +// EndRun ends the test run under run with end's outcome, words, counts, node and output, written once by compare-and-set, +// and ends each of its readers with it but skip, the step whose own end ended it (§725, §726). A run ended already is not +// ended again, and end's ask, the one its caller judged, ends only a run under that ask: a run a rerun asked since is not +// ended by a judgement of the one before. +func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) error { + var r Run + for { + got, rev, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) { + return nil + } + if err != nil { + return fmt.Errorf("the test run %s is not read: %w", run, err) + } + if got.Ended != nil || (end.Ask != "" && got.Ask != end.Ask) { + return nil + } + now := h.now() + got.Ended, got.Outcome, got.Said, got.Tests = &now, end.Outcome, end.Said, end.Tests + if end.Node != "" { + got.Node = end.Node + } + if end.Output != "" { + got.Output = end.Output + } + if _, err := h.TestRuns.Update(run, got, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return fmt.Errorf("the test run %s is not written: %w", run, err) + } + r = got + break + } + if h.Clock != nil { + for _, k := range []string{BoundPrefix + run, UntakenPrefix + run} { + if err := h.Clock.Unbound(k); err != nil { + h.logf("the bound %s of the ended test run is not deleted: %v", k, err) + } + } + } + var first error + for _, reader := range r.Readers { + if skip != nil && reader.Of() == *skip { + continue + } + if err := h.endReader(reader, run, r); err != nil && first == nil { + first = err + } + } + return first +} + +// endReader ends reader's step of run with the run's result (§726): its outcome, words, counts and node, the step naming +// the run in Shared as it did. A step ended already takes nothing, but one the rerun that asked the run marked (A-470d, +// A-470e): it read another ask of the key, and it moves with this one, said again marked a rerun by that caller, naming +// the new ask, its pipeline reopened (§727). +func (h *Hearer) endReader(reader RunReader, run string, r Run) error { + for { + p, rev, err := h.Store.Get(reader.Pipeline) + if err != nil { + return nil // a pipeline gone has no reader left to end + } + i, moved := -1, false + for j, s := range p.Steps { + if s.Artifact != reader.Artifact || sharedRun(s) != run { + continue + } + if open(s) { + i, moved = j, false + } else if reader.Rerun && s.Shared[0].Ask != r.Ask && s.Accepted == nil { + i, moved = j, true + } + } + if i < 0 { + return nil + } + now := h.now() + s := &p.Steps[i] + if moved { + s.Rerun, s.By = true, reader.By + s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: s.Shared[0].AskedBy}} + if p.Outcome != seat.PipelineOutcomeRunning { + p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil + } + } + s.Outcome, s.Why, s.Node, s.Ended, s.Tests = r.Outcome, r.Said, r.Node, &now, nil + if r.Tests != nil { + t := *r.Tests + s.Tests = &t + } + p.Unlogged = append(p.Unlogged, endEntry(reader.Pipeline, *s, now)) + asked, err := askApproval(reader.Pipeline, &p, now, h.Asker) + if err != nil { + return err + } + ended := endPipeline(&p, now) + if _, err := h.Store.Update(reader.Pipeline, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return fmt.Errorf("pipeline %s cannot be written: %w", reader.Pipeline, err) + } + h.Failures.StepEnded(h.Store, p, *s) + if err := h.say(seat.EventStepEnded, p, s); err != nil { + return fmt.Errorf("pipeline %s's %s is written but not said: %w", reader.Pipeline, seat.EventStepEnded, err) + } + h.appendUnlogged(reader.Pipeline) + return h.said(p, asked, ended) + } +} + +// NoReaderLeft is what a test run whose last reader left ends with (A-469z): its ask is dropped, and a later step of the +// candidate starts a new run. +const NoReaderLeft = "no reader is left" + +// leaveRun takes the step of pipeline key that read or asked a test run out of the run's readers (§726): a step superseded +// or cancelled. When no reader is left the run ends `could not run`, naming that (A-469z), and its ask is dropped (§119); +// while one is left the ask stays, so a superseded pipeline never takes away another's run. A step of no test run leaves +// nothing. +func (h *Hearer) leaveRun(key string, s seat.StepResult, why string) { + run := sharedRun(s) + if run == "" || h.TestRuns == nil { + return + } + reader := seat.TestRunReader{Pipeline: key, Artifact: s.Artifact} + for { + r, rev, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) { + return + } + if err != nil { + h.logf("the test run %s is not read, so the step %s of %s does not leave it: %v", run, s.Step, key, err) + return + } + left := r.Readers[:0:0] + for _, o := range r.Readers { + if o.Of() != reader { + left = append(left, o) + } + } + if len(left) == len(r.Readers) { + return + } + r.Readers = left + // The last to leave ends the run in the same write, so a step joining it after that read finds it ended and asks + // its own, never joining a run then ended for having no reader. + last := len(left) == 0 && r.Ended == nil + if last { + now := h.now() + r.Ended, r.Outcome, r.Said = &now, seat.StepResultOutcomeCouldNotRun, NoReaderLeft + } + if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + h.logf("the step %s of %s does not leave the test run %s: %v", s.Step, key, run, err) + return + } + if !last { + return + } + if h.Clock != nil { + for _, k := range []string{BoundPrefix + run, UntakenPrefix + run} { + if err := h.Clock.Unbound(k); err != nil { + h.logf("the bound %s of the ended test run is not deleted: %v", k, err) + } + } + } + if h.DropRun != nil { + if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil { + h.logf("the drop of ask %s of the test run %s, which no reader is left of, is not answered: %v", r.Ask, run, err) + } + } + return + } +} + +// open says whether a step is asked and not ended: running, or a test step waiting for a build node to take its run's ask +// (A-469b). +func open(s seat.StepResult) bool { + return s.Outcome == seat.StepResultOutcomeRunning || s.Outcome == seat.StepResultOutcomeWaitingForABuildNode +} + +// runStarted moves each step reading run from waiting for a build node to running, the holder's ask-taken heard (A-469b), +// and says its step-started again, so the watchman's build-capacity waits let it go (§86, §732). +func (h *Hearer) runStarted(run string, r Run) { + for _, reader := range r.Readers { + for { + p, rev, err := h.Store.Get(reader.Pipeline) + if err != nil { + break + } + var moved []int + for j, s := range p.Steps { + if s.Artifact == reader.Artifact && sharedRun(s) == run && s.Outcome == seat.StepResultOutcomeWaitingForABuildNode { + p.Steps[j].Outcome, p.Steps[j].On = seat.StepResultOutcomeRunning, "" + moved = append(moved, j) + } + } + if len(moved) == 0 { + break + } + if _, err := h.Store.Update(reader.Pipeline, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + h.logf("the steps of %s reading the taken test run %s are not written running: %v", reader.Pipeline, run, err) + break + } + for _, j := range moved { + if err := h.say(seat.EventStepStarted, p, &p.Steps[j]); err != nil { + h.logf("step-started of %s's %s is not said: %v", reader.Pipeline, p.Steps[j].Step, err) + } + } + break + } + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 9db155f9..dcc1ab14 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -4,8 +4,10 @@ import ( "encoding/json" "errors" "fmt" + "slices" "sort" "strconv" + "strings" "time" runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient" @@ -73,8 +75,12 @@ type Starter struct { // Toolchain is the toolchain of a language as the runner heard it agreed (§416, A-104): its language and version; nil when // none was heard, which asks the step with none. Toolchain func(language string) (*nodebuildgo.ToolchainUsed, error) - // Arm arms the bound of an ask put on a build seat's work queue (§535, §647); nil arms none. - Arm Arm + // Holders are the modules the seat mesh-module-assigner's view seat-holders names for a seat, and whether that view's + // first view is read (§535, §647 as A-466y, A-466z amend them); nil, or a view not read yet, judges no seat holderless. + Holders func(seat string) ([]string, bool) + // Ended says a pipeline that ended at its placing (holderless): the step-ended of each step of steps, then its end + // (Hearer.EndedAtPlacing); nil says it in the journal alone. + Ended func(p seat.Pipeline, steps []int) error // Fills refuses a start while a store of the runner is full (§366); nil refuses none. Fills *Fills // Recovered says whether a commit is named a recovery (§81, §86): its steps are asked on the build seats' stream delivery. @@ -83,6 +89,116 @@ type Starter struct { Store Updater // Supersede ends the older running pipelines the pipeline just written supersedes (§31; Hearer.Supersede); nil ends none. Supersede func(newer seat.Pipeline) + // TestRuns is the state test-runs, where a test step's run is shared by every step of its candidate, level and digest + // (§724, §725); nil shares none, each step's ask put. Clock writes a run's untaken bound at its start, Runs holds the + // settings, RunsFill reads test-runs' fill, judged against Runs' cap before a run is written (A-469d), and EndReaders ends + // the steps of a run read ended or never asked. + TestRuns TestRuns + Clock RunClock + Runs RunSettings + RunsFill func() (seat.Fill, error) + EndReaders func(run string, readers []seat.TestRunReader, r seat.TestRun) error +} + +// holderless ends at once, as it is placed, a pipeline with a build step whose seat the seat mesh-module-assigner's view +// seat-holders names no holder for (design 47 G §32, §535, §647 as A-466y and A-466z amend them; A-470g, A-470h): judged only +// once that view's first view is read, never from the picture before it. Each such step ends `could not run` naming the seat, +// every other step not ended ends `not asked`, each with its step log entry, and the pipeline ends `failed` naming the seat; +// none of its steps is asked. It answers the asks still to put and the steps it ended, none when no seat is holderless. +func (s *Starter) holderless(p *seat.Pipeline, asks []Asked, now time.Time) ([]Asked, []int) { + if s.Holders == nil { + return asks, nil + } + var none []string + for _, st := range p.Steps { + if !open(st) || !BoundSeat(st.Seat) || slices.Contains(none, st.Seat) { + continue + } + if holders, read := s.Holders(st.Seat); read && len(holders) == 0 { + none = append(none, st.Seat) + } + } + if len(none) == 0 { + return asks, nil + } + sort.Strings(none) + whys := make([]string, 0, len(none)) + for _, n := range none { + whys = append(whys, fmt.Sprintf("the seat %s has no holder: the seat-holders view names none", n)) + } + var ended []int + for i := range p.Steps { + st := &p.Steps[i] + if !open(*st) && st.Outcome != seat.StepResultOutcomeWaiting { + continue + } + st.Ended, st.Ask, st.Shared = &now, "", nil + if slices.Contains(none, st.Seat) { + st.Outcome, st.Why = seat.StepResultOutcomeCouldNotRun, fmt.Sprintf("the seat %s has no holder: the seat-holders view names none", st.Seat) + } else { + st.Outcome, st.Why = seat.StepResultOutcomeNotAsked, "the pipeline failed at once: "+strings.Join(whys, "; ") + } + p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now)) + ended = append(ended, i) + } + p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeFailed, strings.Join(whys, "; "), &now + p.EndCount++ + return nil, ended +} + +// endedAtPlacing says a pipeline holderless ended at placing (A-470h): each step it ended in its step-ended, then the +// pipeline's end, through Ended; with none given it is said in the journal alone. +func (s *Starter) endedAtPlacing(p seat.Pipeline, steps []int) error { + s.logf("%s ended failed at once: %s (A-470g)", p.Pipeline, p.Why) + if s.Ended == nil { + return nil + } + return s.Ended(p, steps) +} + +// askSteps puts a placed pipeline's asks (§22, §24): a test run another step asked is read, not asked again (share), then +// the kept results and toolchain are carried and each ask is put. +func (s *Starter) askSteps(key, commit string, asks []Asked) error { + placed := asks + asks = s.share(key, asks) + var err error + if len(asks) > 0 { + s.carry(key, asks) + err = askAll(s.Steps, key, asks, s.recovered(key, commit)) + } + s.sayStarted(key, placed) + return err +} + +// sayStarted says step-started for each step of pipeline key that asks were placed for (design 47 G §726, §729; A-469a): +// the step asked, and each step reading the test run it asks, naming the run's key and ask in shared, so the watchman times +// it from that ask (§496, §732). A composition step's, said before any run is chosen, names none (§729). +func (s *Starter) sayStarted(key string, asks []Asked) { + if s.Store == nil || s.Say == nil || len(asks) == 0 { + return + } + ids, runs := map[string]bool{}, map[string]bool{} + for _, a := range asks { + ids[a.ID] = true + if a.Run != "" { + runs[a.Run] = true + } + } + p, _, err := s.Store.Get(key) + if err != nil { + s.logf("the pipeline %s is not read, so its steps' step-started is not said: %v", key, err) + return + } + at := s.now() + for _, st := range p.Steps { + if !open(st) || !(ids[st.Ask] || (st.Ask == "" && runs[sharedRun(st)])) { + continue + } + subject, ev := stepEvent(seat.EventStepStarted, p, st, at) + if err := s.Say(subject, ev); err != nil { + s.logf("step-started of %s's %s %s is not said: %v", key, st.Artifact, st.Step, err) + } + } } func newPipeline(b Begin, key string, now time.Time) seat.Pipeline { @@ -115,11 +231,13 @@ func (s *Starter) Start(b Begin) (bool, error) { } p := newPipeline(b, key, now) var asks []Asked + var ended []int if b.unheld { s.logf("%s starts with no step: the catalog half of %s is not held yet, and its steps are placed once it is (A-336)", key, b.Branch) } else if s.Registration != nil { if reg, ok := s.Registration(b.Module); ok { p.Steps, asks = Place(p, reg, b.Entries, now, s.Name) + asks, ended = s.holderless(&p, asks, now) } else { s.logf("%s has no registration yet: its pipeline starts with no step and waits on the registration of %s, which places them (A-327)", key, b.Module) } @@ -143,9 +261,11 @@ func (s *Starter) Start(b Begin) (bool, error) { if s.Supersede != nil { s.Supersede(p) } + if len(ended) > 0 { + return true, s.endedAtPlacing(p, ended) + } if s.Steps != nil && len(asks) > 0 { - s.carry(key, asks) - if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, b.Commit)); err != nil { + if err := s.askSteps(key, b.Commit, asks); err != nil { return true, err } } @@ -187,24 +307,32 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) { if !ok { continue } - running := map[string]bool{} + // A step that names its test run is asked as the run's asker (share); one that names none was asked before the test + // runs and keeps its own ask, ending by the path it was asked under (§730, A-468k). + running, own := map[string]bool{}, map[string]bool{} for _, st := range p.Steps { - if st.Outcome == seat.StepResultOutcomeRunning && st.Ask != "" { - running[st.Ask] = true + if open(st) && st.Ask != "" { + running[st.Ask], own[st.Ask] = true, len(st.Shared) == 0 } } - _, placed := Place(p, reg, half.Entries, now, s.Name) + _, shared := Place(p, reg, half.Entries, now, s.Name) + _, unshared := place(p, reg, half.Entries, now, s.Name, false) var due []Asked - for _, a := range placed { - if running[a.ID] { + for _, a := range shared { + if running[a.ID] && !own[a.ID] { + due = append(due, a) + } + } + for _, a := range unshared { + if running[a.ID] && own[a.ID] { + a.Run = "" due = append(due, a) } } if len(due) == 0 { continue } - s.carry(key, due) - if err := askAll(s.Steps, s.Arm, key, due, s.recovered(key, p.Commit)); err != nil && first == nil { + if err := s.askSteps(key, p.Commit, due); err != nil && first == nil { first = err } asked += len(due) @@ -284,6 +412,7 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe } var asks []Asked p.Steps, asks = Place(p, r, half.Entries, now, s.Name) + asks, ended := s.holderless(&p, asks, now) if _, err := store.Update(key, p, rev); err != nil { s.logf("%s is written meanwhile, so nothing is placed in it now: %v", key, err) continue @@ -299,9 +428,14 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe s.logf("pipeline-started of %s is not said: %v", key, err) } } + if len(ended) > 0 { + if err := s.endedAtPlacing(p, ended); err != nil { + note(err) + } + continue + } if s.Steps != nil && len(asks) > 0 { - s.carry(key, asks) - if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil { + if err := s.askSteps(key, p.Commit, asks); err != nil { note(err) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index a1c7a3c2..63caa13f 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -26,6 +26,9 @@ type StepSeat struct { Prefix string // Decode reads the seat's step-ended body; nil is the build seats' one form. Decode func([]byte) (StepEnded, error) + // Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for + // a seat whose ends name none. + Shared func([]byte) []seat.Shared } // StepSeats are the three build seats, from their contracts. @@ -36,7 +39,25 @@ var StepSeats = []StepSeat{ // The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99). {Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded}, // The seat mesh-wants answers the composition step (§22, §24). - {Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded}, + {Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared}, +} + +// wantsShared is the composition runs a mesh-wants step-ended names, one per machine (design 47 G §723, §729; A-468d, +// A-469t): the same form, under the same name. +func wantsShared(raw []byte) []seat.Shared { + var w wants.StepEnded + if json.Unmarshal(raw, &w) != nil || len(w.Shared) == 0 { + return nil + } + out := make([]seat.Shared, 0, len(w.Shared)) + for _, x := range w.Shared { + e := seat.Shared{Machine: x.Machine, Run: x.Run, Ask: x.Ask} + if x.AskedBy != nil { + e.AskedBy = &seat.SharedAskedBy{Pipeline: x.AskedBy.Pipeline, Artifact: x.AskedBy.Artifact} + } + out = append(out, e) + } + return out } // decodeWantsEnded reads the composition step's end of the seat mesh-wants as the form the hearer holds each end to; a @@ -118,6 +139,26 @@ type Hearer struct { Fills *Fills // Widens computes the widens a pipeline-ended carries (§71, A-207); nil carries none. Widens func(seat.Pipeline) []seat.Widening + // TestRuns is the state test-runs and Clock its bound keys (§725); Runs the settings of its runs; DropRun drops a + // run's ask when its run ends unfinished (§119). A nil TestRuns keeps no test run. + TestRuns TestRuns + Clock RunClock + Runs RunSettings + DropRun Drop + // RunOutputs keeps a test run's output in test-runs-objects (§725, A-466v). + RunOutputs RunObjects +} + +// EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps +// of a run that was never asked. +func (h *Hearer) EndReaders(run string, readers []seat.TestRunReader, r seat.TestRun) error { + var first error + for _, reader := range readers { + if err := h.endReader(RunReader{Pipeline: reader.Pipeline, Artifact: reader.Artifact}, run, Run{TestRun: r}); err != nil && first == nil { + first = err + } + } + return first } func (h *Hearer) now() time.Time { @@ -132,6 +173,13 @@ func (h *Hearer) now() time.Time { // another seat, commit, fingerprint, module or step (ADR 0666 §21; the bus grants step-ended. to that node's // holder alone, so the subject's node is the account's). A step already ended takes nothing (the bus gives an event again). func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pipeline, error) { + return h.OnStepEndedShared(bs, subject, ev, nil) +} + +// OnStepEndedShared is OnStepEnded of an end naming the runs it read in shared (A-468d): kept on the step's result. The end +// of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726, +// A-469m). +func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) { node := strings.TrimPrefix(subject, bs.Prefix) if node == subject || node == "" || strings.Contains(node, ".") { return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat) @@ -153,7 +201,15 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi case ev.Commit != p.Commit || ev.Fingerprint != p.Fingerprint || ev.Module != p.Module || ev.Step != p.Steps[i].Step: return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s at %s (%s)", ErrRefused, ev.ID, ev.Step, ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint) - case p.Steps[i].Outcome != seat.StepResultOutcomeRunning: + case !open(p.Steps[i]): + if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil { + if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID { + // The asking step's end was written and its run's was not (a write failed): the run ends now, heard again. + return p, h.endRunOf(p, p.Steps[i], ev) + } + // The run ended before its ask's end came (its bound, a drop): the end is ignored and said (§720, §725). + h.logf("the step-ended of ask %s is ignored: its test run %s ended before it (%s)", ev.ID, run, p.Steps[i].Why) + } return p, nil case !endOutcome(ev.Outcome): return p, fmt.Errorf("%w: ask %s ended %q, not an outcome of step-ended", ErrRefused, ev.ID, ev.Outcome) @@ -161,6 +217,9 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi now := h.now() s := &p.Steps[i] s.Outcome, s.Node, s.Why, s.Rerun, s.By, s.Ended = ev.Outcome, node, ev.Said, ev.Rerun, ev.By, &now + if len(shared) > 0 { + s.Shared = shared + } if ev.Tests != nil { s.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration} } @@ -177,6 +236,8 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi } unlogged := endEntry(ev.Pipeline, *s, now, keptKeys...) p.Unlogged = append(p.Unlogged, unlogged) + // The steps of this pipeline reading the run end in the same write, so the pipeline ends once (§726, §727). + readers := h.sameRunReaders(&p, i, ev, now) from := now if ev.Ended != nil { from = ev.Ended.UTC() @@ -191,6 +252,9 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi } else if err != nil { return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err) } + // The run's end is written as its step-ended is heard, before any finding the step says, so no wait elsewhere (a + // finding's event the bus is slow to take) delays it past its bound (A-470o). + runErr := h.endRunOf(p, *s, ev) if h.Keep != nil { for _, k := range ev.Kept { kept := seat.KeptResult{Digest: k.Digest, Level: k.Level, Package: k.Package, Pipeline: k.Pipeline, LastRead: &now} @@ -209,14 +273,94 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi if err := h.say(seat.EventStepEnded, p, s); err != nil { return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err) } + for _, j := range readers { + if err := h.say(seat.EventStepEnded, p, &p.Steps[j]); err != nil { + return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err) + } + } // The step's end is written and said before its log entry is appended, so a log that refuses ends no step (§486, §606). h.appendUnlogged(ev.Pipeline) h.Ager.Age() h.Fills.Check() - return p, h.said(p, asked, ended) + if err := h.said(p, asked, ended); err != nil { + return p, err + } + return p, runErr } } +// sameRunReaders ends, in p, every other step reading the test run step i asked with ev, its ask's end: a running reader, +// and one the rerun that asked the run marked, which moves (A-470d, A-470e), said again marked a rerun by that caller. It +// answers their indexes. +func (h *Hearer) sameRunReaders(p *seat.Pipeline, i int, ev StepEnded, now time.Time) []int { + asker := p.Steps[i] + run := sharedRun(asker) + if run == "" || h.TestRuns == nil || asker.Ask != ev.ID { + return nil + } + marks := map[string]RunReader{} + if r, _, err := h.TestRuns.Get(run); err == nil { + for _, o := range r.Readers { + if o.Pipeline == p.Pipeline { + marks[o.Artifact] = o + } + } + } + var out []int + for j := range p.Steps { + s := &p.Steps[j] + if j == i || sharedRun(*s) != run { + continue + } + switch m := marks[s.Artifact]; { + case open(*s) && s.Ask == "": + case m.Rerun && s.Shared[0].Ask != ev.ID && s.Accepted == nil: + s.Rerun, s.By = true, m.By + s.Shared = []seat.Shared{{Run: run, Ask: ev.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: asker.Artifact}}} + default: + continue + } + s.Outcome, s.Why, s.Node, s.Ended, s.Tests = asker.Outcome, asker.Why, asker.Node, &now, nil + if asker.Tests != nil { + t := *asker.Tests + s.Tests = &t + } + p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *s, now)) + out = append(out, j) + } + return out +} + +// endRunOf ends the test run step s of p asked, with the end ev of its ask (§725, §726): the run's output kept in +// test-runs-objects under the run's key, its outcome, words, counts and node written, and every other reader ended with it. +// A step that asked no run ends nothing more. +func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) error { + run := sharedRun(s) + if run == "" || h.TestRuns == nil || s.Ask != ev.ID { + return nil + } + end := seat.TestRun{Ask: ev.ID, Outcome: ev.Outcome, Said: ev.Said, Node: ev.Node} + if ev.Tests != nil { + end.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration} + } + if h.RunOutputs != nil { + // The run's output object holds the packages it ran with their digests and the build seat's output (§725). + out := RunOutput{Output: ev.Output, Packages: []seat.TestRunPackage{}} + for _, k := range append(append([]nodebuildgo.KeptResult(nil), ev.Kept...), ev.Carried...) { + out.Packages = append(out.Packages, seat.TestRunPackage{Package: k.Package, Digest: k.Digest}) + } + raw, err := json.Marshal(out) + if err == nil { + err = h.RunOutputs.Put(run, raw) + } + if err != nil { + return fmt.Errorf("the output of the test run %s cannot be kept: %w", run, err) + } + end.Output = run + } + return h.EndRun(run, end, &seat.TestRunReader{Pipeline: p.Pipeline, Artifact: s.Artifact}) +} + // said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall. func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error { if asked { @@ -300,8 +444,8 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error at := h.now() switch event { case seat.EventStepEnded, seat.EventStepStarted: - return h.Say(subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, - Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: *step}) + _, ev := stepEvent(event, p, *step, at) + return h.Say(subject, ev) case seat.EventPipelineEnded: e := pipelineEnded(p) if h.Widens != nil { @@ -314,17 +458,46 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error } } +// stepEvent is a step's step-started or step-ended of p, on the runner's subject for it: its result as it stands, with +// its shared when it reads a run (§729; A-468e, A-469q). +func stepEvent(event string, p seat.Pipeline, step seat.StepResult, at time.Time) (string, seat.StepEvent) { + last := p.Commit + if p.Group != "" { + last = p.Group + } + if event == seat.EventStepStarted { + // A step-started names the run and its ask, never the step that asked it (A-469q). + var shared []seat.Shared + for _, x := range step.Shared { + x.AskedBy = nil + shared = append(shared, x) + } + step.Shared = shared + } + subject := "mesh.seat." + seat.Seat + ".event." + event + "." + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Branch) + "." + keys.EscapeName(last) + return subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, + Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: step} +} + // The test budget per catalog module (design 47 G §24, §419; the test budget): at most 500 unit and 50 integration tests. const ( budgetUnit = 500 budgetIntegration = 50 ) -// testCounts is the module's test counts, the sum of its test steps' own counts; nil when no step counted any. +// testCounts is the module's test counts, the sum of its test steps' own counts, each test run counted once however many +// steps read it (§726, A-466i); nil when no step counted any. func testCounts(steps []seat.StepResult) *seat.TestCounts { var t seat.TestCounts counted := false + runs := map[string]bool{} for _, s := range steps { + if run := sharedRun(s); run != "" && s.Seat != wants.Seat { + if runs[run] { + continue + } + runs[run] = true + } if s.Tests != nil { t.Unit, t.Integration, counted = t.Unit+s.Tests.Unit, t.Integration+s.Tests.Integration, true } @@ -380,7 +553,7 @@ func endPipeline(p *seat.Pipeline, now time.Time) bool { // An optional step still waiting (a code review nobody has given) holds nothing: only a required one holds the // pipeline's end (A-104). continue - case s.Outcome == seat.StepResultOutcomeWaiting, s.Outcome == seat.StepResultOutcomeRunning: + case s.Outcome == seat.StepResultOutcomeWaiting, open(s): return false } if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried { @@ -401,3 +574,22 @@ func endPipeline(p *seat.Pipeline, now time.Time) bool { } return true } + +// EndedAtPlacing says a pipeline its starter ended at placing, a seat with no holder (A-470g, A-470h): the step-ended of each +// step of steps, its step log entries appended, the head's finding pipeline-failed (§33), then its pipeline-ended. +func (h *Hearer) EndedAtPlacing(p seat.Pipeline, steps []int) error { + var first error + for _, i := range steps { + if err := h.say(seat.EventStepEnded, p, &p.Steps[i]); err != nil && first == nil { + first = fmt.Errorf("pipeline %s's %s is written but not said: %w", p.Pipeline, seat.EventStepEnded, err) + } + } + h.appendUnlogged(p.Pipeline) + if len(steps) > 0 { + h.Failures.StepEnded(h.Store, p, p.Steps[steps[0]]) + } + if err := h.said(p, false, true); err != nil && first == nil { + first = err + } + return first +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go index edd4e860..114411d9 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps_test.go @@ -302,49 +302,6 @@ func TestAnOptionalStepStillWaitingHoldsNoEnd(t *testing.T) { } } -// An ask no holder took within its bound ends its step `could not run` naming the seat, and with the rest ended the pipeline -// failed (design 47 G §32, §535, §647, A-104); a step that ended, or ask nobody placed, ends nothing. -func TestAnAskNoHolderTookEndsItsStepNamingTheSeat(t *testing.T) { - h, store, said, _, _, _ := fixture() - e := store[pkey] - e.p.Steps[1].Outcome = seat.StepResultOutcomePassed // the unit step ended; only lint is waiting for a holder - e.p.Steps[2].Outcome = seat.StepResultOutcomeNotAsked - store[pkey] = e - if err := h.NotTaken("nobody-asked", AskBound); err != nil || store[pkey].p.Steps[0].Outcome != seat.StepResultOutcomeRunning { - t.Fatalf("an ask no step names ended something: %v", err) - } - if err := h.NotTaken("a1", AskBound); err != nil { - t.Fatal(err) - } - p := store[pkey].p - if p.Steps[0].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(p.Steps[0].Why, nodebuildgo.Seat) || - p.Outcome != seat.PipelineOutcomeFailed || !strings.Contains(strings.Join(*said, " "), seat.EventPipelineEnded) { - t.Errorf("step %+v, pipeline %s (%s), said %v: want could not run naming %s and a failed pipeline", p.Steps[0], p.Outcome, p.Why, *said, nodebuildgo.Seat) - } - // Its end taken once: the bound's expiry heard again changes nothing. - n := len(*said) - if err := h.NotTaken("a1", AskBound); err != nil || len(*said) != n { - t.Errorf("a second expiry said %d more events: %v", len(*said)-n, err) - } -} - -// The bound is armed before the ask is put, only for the build seats whose holder says ask-taken, and an ask whose bound is -// not armed is not put (§535, §647). -func TestTheBoundIsArmedBeforeAnAskIsPut(t *testing.T) { - var order []string - asker := askerFunc(func(accept, id string, _ any) error { order = append(order, "put "+id); return nil }) - arm := Arm(func(ask string) error { order = append(order, "arm "+ask); return nil }) - asks := []Asked{{Seat: nodebuildgo.Seat, ID: "step-1", Ask: 1}, {Seat: wants.Seat, ID: "step-2", Ask: 2}} - if err := askAll(asker, arm, "k", asks, false); err != nil || strings.Join(order, ",") != "arm step-1,put step-1,put step-2" { - t.Errorf("order %v, %v; want the build seat's bound armed first and mesh-wants' ask with none", order, err) - } - order = nil - failing := Arm(func(string) error { return errors.New("bucket gone") }) - if err := askAll(asker, failing, "k", asks[:1], false); err == nil || len(order) != 0 { - t.Errorf("an ask whose bound cannot be armed was put or said nothing: %v %v", order, err) - } -} - type askerFunc func(accept, id string, body any) error func (f askerFunc) AskStep(accept, id string, body any) error { return f(accept, id, body) } @@ -359,7 +316,7 @@ func TestARecoverysStepsAreAskedOnTheStreamDelivery(t *testing.T) { }{{true, "mesh.seat.node-build-go.accept.delivery.mesh-pipeline-runner," + wants.AcceptSubjectStep}, {false, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner," + wants.AcceptSubjectStep}} { var to []string asker := askerFunc(func(accept, _ string, _ any) error { to = append(to, accept); return nil }) - if err := askAll(asker, nil, "k", asks, c.recovery); err != nil || strings.Join(to, ",") != c.want { + if err := askAll(asker, "k", asks, c.recovery); err != nil || strings.Join(to, ",") != c.want { t.Errorf("recovery %v: asked on %v (%v), want %s", c.recovery, to, err, c.want) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go index a729b552..30be4d2f 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go @@ -30,7 +30,7 @@ func (h *Hearer) Taken(ask, node string) error { break } i := stepOfAsk(p, ask) - if i < 0 || p.Steps[i].Outcome != seat.StepResultOutcomeRunning || p.Steps[i].Node == node { + if i < 0 || !open(p.Steps[i]) || p.Steps[i].Node == node { break } p.Steps[i].Node = node @@ -49,10 +49,10 @@ func (h *Hearer) Taken(ask, node string) error { // whose fingerprint newer's commit changed, and for a publish's pipeline each an older publish of the package started // (design 47 G §31, §119; ADR 0666 §9; A-202): the pipeline and each step not ended // `superseded`, naming the newer commit, and it says pipeline-ended. A module the newer commit left as it was finishes its -// pipeline. The ask of each superseded step on a build seat that serves `drop` is dropped (drop; its bound deleted through -// unbound); a step on mesh-builds or mesh-wants, whose seats serve no drop, is only ended superseded (A-182). A run on +// pipeline. The ask of each superseded step on a build seat that serves `drop` is dropped (drop); a step of a test run leaves +// it, its ask dropped only with its last reader (§726); a step on mesh-builds or mesh-wants, whose seats serve no drop, is only ended superseded (A-182). A run on // request's pipeline neither supersedes nor is superseded. It says how many pipelines it ended. -func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop, unbound func(ask string) error) (int, error) { +func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop) (int, error) { if newer.Run != "" || newer.Source <= 0 { return 0, nil } @@ -67,7 +67,7 @@ func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop, unbound func(ask stri if key == newer.Pipeline { continue } - done, err := h.supersedeOne(key, newer, why, drop, unbound) + done, err := h.supersedeOne(key, newer, why, drop) if err != nil && first == nil { first = err } @@ -99,7 +99,7 @@ func supersededWhy(newer seat.Pipeline) string { return fmt.Sprintf("the newer commit %s of %s changed %s's fingerprint", newer.Commit, newer.Branch, newer.Module) } -func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop Drop, unbound func(string) error) (bool, error) { +func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop Drop) (bool, error) { for { p, rev, err := h.Store.Get(key) if err != nil || !superseded(p, newer) { @@ -110,10 +110,11 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop var changed []int for i := range p.Steps { s := &p.Steps[i] - if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting { + if !open(*s) && s.Outcome != seat.StepResultOutcomeWaiting { continue } - if s.Ask != "" && BoundSeat(s.Seat) { + // A step of a test run leaves it: its ask is dropped only with its last reader (§726, leaveRun). + if s.Ask != "" && BoundSeat(s.Seat) && sharedRun(*s) == "" { drops = append(drops, *s) } s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeSuperseded, why, ptrTime(now) @@ -136,17 +137,15 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop if len(changed) > 0 { h.appendUnlogged(key) } + for _, i := range changed { + h.leaveRun(key, p.Steps[i], why) + } for _, s := range drops { if drop != nil { if err := drop(s.Seat, s.Node, s.Ask, why); err != nil { h.logf("the drop of ask %s of the superseded %s is not answered: %v", s.Ask, key, err) } } - if unbound != nil { - if err := unbound(s.Ask); err != nil { - h.logf("the bound of ask %s of the superseded %s is not dropped: %v", s.Ask, key, err) - } - } } if err := h.said(p, false, true); err != nil && first == nil { first = err diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/supersede_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/supersede_test.go index 8490be86..95a06e7c 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/supersede_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/supersede_test.go @@ -29,7 +29,6 @@ func TestSupersede(t *testing.T) { } type dropped struct{ seat, node, ask string } var drops []dropped - var unbound []string drop := func(s, node, ask, why string) error { if !strings.Contains(why, "c2") { t.Errorf("the drop of %s names no newer commit: %q", ask, why) @@ -37,7 +36,6 @@ func TestSupersede(t *testing.T) { drops = append(drops, dropped{s, node, ask}) return nil } - unbind := func(ask string) error { unbound = append(unbound, ask); return nil } newer := func(key string, source int64, fp string, run string, pub *seat.Published) seat.Pipeline { return seat.Pipeline{Pipeline: key, Branch: "main", Source: source, Module: "app", Commit: "c2", Fingerprint: fp, Run: run, Published: pub, Outcome: seat.PipelineOutcomeRunning} @@ -50,12 +48,12 @@ func TestSupersede(t *testing.T) { {"a run on request", newer("main.r1.app", 6, "src2:bb", "r1", nil)}, {"an older source revision", newer("main.4.app", 4, "src2:bb", "", nil)}, } { - if n, err := h.Supersede(c.newer, drop, unbind); n != 0 || err != nil { + if n, err := h.Supersede(c.newer, drop); n != 0 || err != nil { t.Errorf("%s superseded %d (%v)", c.name, n, err) } } *said = nil - if n, err := h.Supersede(newer("main.6.app", 6, "src2:bb", "", nil), drop, unbind); n != 1 || err != nil { + if n, err := h.Supersede(newer("main.6.app", 6, "src2:bb", "", nil), drop); n != 1 || err != nil { t.Fatalf("a changed fingerprint superseded %d (%v)", n, err) } p, _, _ := store.Get(pkey) @@ -68,13 +66,13 @@ func TestSupersede(t *testing.T) { } } want := []dropped{{nodebuildgo.Seat, "n7", "a1"}, {nodebuildgo.Seat, "", "a2"}} - if !slices.Equal(drops, want) || !slices.Equal(unbound, []string{"a1", "a2"}) { - t.Errorf("dropped %+v and unbound %v, want %+v (the mesh-builds step only ended)", drops, unbound, want) + if !slices.Equal(drops, want) { + t.Errorf("dropped %+v, want %+v (the mesh-builds step only ended)", drops, want) } if n := len(*said); n != 5 || (*said)[4] != seat.EventPipelineEnded+".app.main.c1" { t.Errorf("said %v, want four step-ended then pipeline-ended", *said) } - if n, _ := h.Supersede(newer("main.7.app", 7, "src2:cc", "", nil), drop, unbind); n != 0 { + if n, _ := h.Supersede(newer("main.7.app", 7, "src2:cc", "", nil), drop); n != 0 { t.Errorf("a superseded pipeline was superseded again: %d", n) } @@ -82,10 +80,10 @@ func TestSupersede(t *testing.T) { pub := memStore{"main.5.lib": {p: seat.Pipeline{Pipeline: "main.5.lib", Branch: "main", Source: 5, Module: "lib", Fingerprint: "src2:aa", Outcome: seat.PipelineOutcomeRunning, Published: &seat.Published{Artifact: "core:pkg", Version: "v1"}}, rev: 1}} h.Store = pub - if n, _ := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v1"}), drop, unbind); n != 0 { + if n, _ := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v1"}), drop); n != 0 { t.Errorf("a publish of the same version superseded %d", n) } - if n, err := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v2"}), drop, unbind); n != 1 || err != nil { + if n, err := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v2"}), drop); n != 1 || err != nil { t.Fatalf("a newer publish superseded %d (%v)", n, err) } if p, _, _ := pub.Get("main.5.lib"); p.Outcome != seat.PipelineOutcomeSuperseded || !strings.Contains(p.Why, "v2") { @@ -105,9 +103,6 @@ func TestBoundDrop(t *testing.T) { if _, _, ok := ParseDropBound("step-ab"); ok { t.Error("an ask's bound was read as a drop's") } - if IsAskKey(DropBoundKey("s", "step-ab")) { - t.Error("a drop's bound was taken for an ask's") - } err := BoundDrop(func(string, string, string, string) error { return nil }, arm, done)(nodebuildgo.Seat, "n1", "step-ab", "why") if err != nil || len(armed) != 1 || armed[0] != "drop."+nodebuildgo.Seat+".step-ab" || len(answered) != 1 { t.Errorf("an answered drop: %v, armed %v, answered %v", err, armed, answered) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go new file mode 100644 index 00000000..494a2cb7 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go @@ -0,0 +1,374 @@ +package pipeline + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "sort" + "strings" + + nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// TestRuns is the runner's state test-runs (design 47 G §725): one key per test run of a candidate, level and tested +// packages, written create-only by the step that asks the run and changed by compare-and-set. +type TestRuns interface { + // Create writes key when it is absent; ErrExists when it is written already. + Create(key string, r Run) error + // Get reads key and its revision; ErrNoRun when it is absent. + Get(key string) (Run, uint64, error) + // Update writes key at the revision read; ErrChanged when another write came between. + Update(key string, r Run, revision uint64) (uint64, error) + // Delete deletes key at the revision read; ErrChanged when another write came between, a key gone is no fault. No + // write of a run puts a value judged from an earlier read over a later one (the kind of the stale write). + Delete(key string, revision uint64) error +} + +// Run is a key's value in test-runs (§725): the test run, its readers each marked whether a rerun's new run moves it and by +// whom (A-470d), so the marks are in the key and a restart takes them up. +type Run struct { + seat.TestRun + Readers []RunReader `json:"readers,omitempty"` +} + +// RunReader is a step reading a test run, its pipeline key and artifact (§725), marked rerun with the caller of the rerun +// whose new run moves it (A-470d). +type RunReader struct { + Pipeline string `json:"pipeline"` + Artifact string `json:"artifact"` + Rerun bool `json:"rerun,omitempty"` + By string `json:"by,omitempty"` +} + +// Of is the reader as the seat names it. +func (r RunReader) Of() seat.TestRunReader { + return seat.TestRunReader{Pipeline: r.Pipeline, Artifact: r.Artifact} +} + +// Answer is the run as the seat's read answers it, its readers named by pipeline and artifact. +func (r Run) Answer() seat.TestRun { + t := r.TestRun + t.Readers = nil + for _, o := range r.Readers { + t.Readers = append(t.Readers, o.Of()) + } + return t +} + +// ErrNoRun is a Get of a test-runs key that is not written. +var ErrNoRun = errors.New("no test run under the key") + +// CandidateOf is a pipeline's candidate in A-226's form (§724): `run-` for a run on request, else `.` +// of the catalog graph revision it was placed from, built by the node-engine seat's client as the assigner builds it. A +// pipeline naming neither has no candidate, and its steps share no run. +func CandidateOf(p seat.Pipeline) (string, bool) { + var key string + var err error + if p.Run != "" { + key, err = nec.RunKey(p.Run) + } else { + key, err = nec.RevisionKey(p.Branch, p.Revision) + } + return key, err == nil +} + +// ShareDigest is a test step's share digest (design 47 G §724, A-469c): the sha256 of the module's build-input fingerprint, +// the level, the step's status in the policy and the step's fetch (its directory and its sorted paths), what the runner +// holds, so two artifacts with one fetch meet and two with different fetches do not. +func ShareDigest(fingerprint, level, status string, fetch nodebuildgo.Fetch) string { + paths := append([]string(nil), fetch.Paths...) + sort.Strings(paths) + h := sha256.New() + for _, part := range []string{fingerprint, level, status, fetch.Dir} { + h.Write([]byte(part + "\n")) + } + h.Write([]byte(strings.Join(paths, "\n"))) + return hex.EncodeToString(h.Sum(nil)) +} + +// TestRunKey is a test run's key in test-runs (§725): `..`, the level and the digest each +// escaped as a key's name, the candidate already in its key form. +func TestRunKey(candidate, level, digest string) string { + return candidate + "." + keys.EscapeName(level) + "." + keys.EscapeName(digest) +} + +// sharedRun is the one run a step's Shared names, empty when it names none. +func sharedRun(s seat.StepResult) string { + if len(s.Shared) == 0 { + return "" + } + return s.Shared[0].Run +} + +// readsRun says whether step s of a pipeline reads run as one step that does not ask it: it names the run and keeps no ask +// of its own (A-469m). +func readsRun(s seat.StepResult, run string) bool { + return run != "" && sharedRun(s) == run && s.Ask == "" +} + +// addReaders adds each reader not in r's readers yet, in order. +func addReaders(r *Run, readers []seat.TestRunReader) { + for _, n := range readers { + have := false + for _, o := range r.Readers { + have = have || o.Of() == n + } + if !have { + r.Readers = append(r.Readers, RunReader{Pipeline: n.Pipeline, Artifact: n.Artifact}) + } + } +} + +// askerOf is the step that asked the run under ask: the step of a pipeline the runner keeps whose own ask it is. +func askerOf(store Updater, ask string) (*seat.SharedAskedBy, error) { + if store == nil || ask == "" { + return nil, nil + } + pkeys, err := store.Keys("*.*.*") + if err != nil { + return nil, fmt.Errorf("the pipelines are not listed: %w", err) + } + for _, key := range pkeys { + p, _, err := store.Get(key) + if err != nil { + continue + } + if i := stepOfAsk(p, ask); i >= 0 { + return &seat.SharedAskedBy{Pipeline: key, Artifact: p.Steps[i].Artifact}, nil + } + } + return nil, nil +} + +// share writes the test run of each ask that asks one (§724, §725): the step that writes the key create-only asks the run; +// one that finds the key written by another step reads it, its pipeline's steps of that run then keeping no ask of their +// own and naming the run's ask and its asker (A-469m), and its ask is not put. A run ended `could not run` is never read: +// it is replaced by compare-and-set and asked again. A key written by this very ask (a start asked again) asks it again, +// kept once by its id. It answers the asks still to put. +func (s *Starter) share(key string, asks []Asked) []Asked { + if s.TestRuns == nil { + return asks + } + var out []Asked + joined := map[string]Run{} + for _, a := range asks { + if a.Run == "" { + out = append(out, a) + continue + } + run, ok, err := s.claimRun(a) + var full *fullErr + switch { + case errors.As(err, &full): + // At the cap no run is asked: the steps end could not run, naming the fill (§725, A-469d). + s.logf("the test run %s of %s is not asked: %v", a.Run, key, err) + if s.EndReaders != nil { + if err := s.EndReaders(a.Run, a.Readers, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: err.Error()}); err != nil { + s.logf("the steps of the test run %s, not asked, are not ended: %v", a.Run, err) + } + } + case err != nil: + // A run that cannot be written or read is asked as the step's own, said: slower, never lost. + s.logf("the test run %s of %s is not written, so the step is asked on its own: %v", a.Run, key, err) + out = append(out, a) + case ok: + out = append(out, a) + default: + joined[a.Run] = run + } + } + if len(joined) > 0 { + s.readOthers(key, joined) + // A run that ended within the candidate is read at once (§724): its readers end with its result. + for run, r := range joined { + if r.Ended == nil || s.EndReaders == nil { + continue + } + var readers []seat.TestRunReader + for _, a := range asks { + if a.Run == run { + readers = a.Readers + } + } + if err := s.EndReaders(run, readers, r.TestRun); err != nil { + s.logf("the steps of %s reading the ended test run %s are not ended: %v", key, run, err) + } + } + } + return out +} + +// fullErr is a run not written because test-runs is at its cap (A-469d), naming the fill. +type fullErr struct{ fill seat.Fill } + +func (e *fullErr) Error() string { + return fmt.Sprintf("the state %s holds %d of its %d bytes, so no test run is asked (test-runs-cap-mib)", e.fill.Store, e.fill.Bytes, e.fill.Cap) +} + +// TestRunsFill is test-runs' fill against its cap setting test-runs-cap-mib, the cap that refuses a new run (A-469d, A-470i), +// never the manifest's ceiling. +func (s *Starter) TestRunsFill() (seat.Fill, error) { + if s.RunsFill == nil { + return seat.Fill{}, errors.New("the runner reads no fill of test-runs") + } + f, err := s.RunsFill() + if err != nil { + return seat.Fill{}, err + } + f.Cap = s.Runs.CapBytes + if f.Cap <= 0 { + f.Cap = DefaultRunSettings.CapBytes + } + return f, nil +} + +// atCap is the error of test-runs at its cap setting, nil below it or when its fill is not read (said). +func (s *Starter) atCap() error { + if s.RunsFill == nil { + return nil + } + f, err := s.TestRunsFill() + if err != nil { + s.logf("the fill of test-runs is not read, so its cap is not judged: %v", err) + return nil + } + if f.Bytes >= f.Cap { + return &fullErr{fill: f} + } + return nil +} + +// started writes a new run and its untaken bound (§725, A-470). +func (s *Starter) started(a Asked, write func() error) error { + if err := s.atCap(); err != nil { + return err + } + if err := write(); err != nil { + return err + } + if s.Clock != nil { + untaken := s.Runs.Untaken + if untaken <= 0 { + untaken = DefaultRunSettings.Untaken + } + if err := armBound(s.Clock, s.TestRuns, UntakenPrefix+a.Run, a.Run, a.ID, untaken); err != nil { + s.logf("the untaken bound of the test run %s is not written: %v", a.Run, err) + } + } + return nil +} + +// claimRun writes a's test run create-only; ok says the ask is put (the key is a's own), else run is the run it reads. +func (s *Starter) claimRun(a Asked) (Run, bool, error) { + now := s.now() + value := Run{TestRun: seat.TestRun{Level: a.Level, Digest: a.Digest, Seat: a.Seat, Ask: a.ID, Started: &now}} + addReaders(&value, a.Readers) + for { + err := s.started(a, func() error { return s.TestRuns.Create(a.Run, value) }) + if err == nil { + return value, true, nil + } + var full *fullErr + if errors.As(err, &full) { + return Run{}, false, err + } + if !errors.Is(err, ErrExists) { + return Run{}, false, err + } + r, rev, err := s.TestRuns.Get(a.Run) + if errors.Is(err, ErrNoRun) { + continue // deleted meanwhile: written again + } + if err != nil { + return Run{}, false, err + } + if r.Ask == a.ID { + return r, true, nil + } + if r.Ended != nil && r.Outcome == seat.StepResultOutcomeCouldNotRun { + err := s.started(a, func() error { + _, err := s.TestRuns.Update(a.Run, value, rev) + return err + }) + if errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return Run{}, false, err + } + return value, true, nil + } + addReaders(&r, a.Readers) + if _, err := s.TestRuns.Update(a.Run, r, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return Run{}, false, err + } + return r, false, nil + } +} + +// readOthers writes pipeline key's steps that read another step's run: no ask of their own, the run's ask and its asker +// in Shared (A-469m, A-469q). +func (s *Starter) readOthers(key string, joined map[string]Run) { + if s.Store == nil { + return + } + by := map[string]*seat.SharedAskedBy{} + for run, r := range joined { + asker, err := askerOf(s.Store, r.Ask) + if err != nil { + s.logf("the step that asked the test run %s is not read: %v", run, err) + } + by[run] = asker + } + for { + p, rev, err := s.Store.Get(key) + if err != nil { + s.logf("the pipeline %s is not read, so its steps reading other runs are not written: %v", key, err) + return + } + for i := range p.Steps { + st := &p.Steps[i] + r, ok := joined[sharedRun(*st)] + if !ok || !open(*st) { + continue + } + if r.Node != "" { + st.Outcome, st.On = seat.StepResultOutcomeRunning, "" // its run's ask is taken already (A-469b) + } + st.Ask = "" + st.Shared = []seat.Shared{{Run: sharedRun(*st), Ask: r.Ask, AskedBy: by[sharedRun(*st)]}} + } + if _, err := s.Store.Update(key, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + s.logf("the pipeline %s's steps reading other runs are not written: %v", key, err) + } + return + } +} + +// joinRun adds reader to run's readers by compare-and-set. +func (s *Starter) joinRun(run string, reader seat.TestRunReader) { + if s.TestRuns == nil { + return + } + for { + r, rev, err := s.TestRuns.Get(run) + if err != nil { + s.logf("the test run %s is not read, so %s does not join it: %v", run, reader.Pipeline, err) + return + } + addReaders(&r, []seat.TestRunReader{reader}) + if _, err := s.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + s.logf("%s does not join the test run %s: %v", reader.Pipeline, run, err) + } + return + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_read.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_read.go new file mode 100644 index 00000000..b725c219 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_read.go @@ -0,0 +1,176 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// RunOutput is a test run's output object in test-runs-objects (§725): the packages the run ran with their digests, the +// passing ones it kept and the carried ones it skipped, and the build seat's output. +type RunOutput struct { + Packages []seat.TestRunPackage `json:"packages"` + Output string `json:"output,omitempty"` +} + +// Gone is what a run whose output object is gone answers in place of its packages (§725, §733). +const Gone = "the run's output is gone" + +// RunsRead answers the seat verb test-runs (design 47 G §728; A-469u) from the runner's state test-runs and its object +// store: each run with its packages read from its output object, and the state's fill. +type RunsRead struct { + Runs Kept + // Object reads an object of test-runs-objects; ErrNoRun when it is gone. + Object func(name string) ([]byte, error) + Fill func() (seat.Fill, error) +} + +// TestRuns answers each test run of a candidate, a level, a digest or a module, or every one. +func (r RunsRead) TestRuns(in seat.TestRunsInput) (seat.TestRunsAnswer, error) { + all, err := r.Runs.Keys() + if err != nil { + return seat.TestRunsAnswer{}, fmt.Errorf("the test runs are not listed: %w", err) + } + sort.Strings(all) + a := seat.TestRunsAnswer{Runs: []seat.TestRun{}} + for _, key := range all { + if IsRunBound(key) || (in.Candidate != "" && !strings.HasPrefix(key, in.Candidate+".")) { + continue + } + raw, err := r.Runs.Value(key) + if err != nil || raw == nil { + continue + } + var run Run + if err := json.Unmarshal(raw, &run); err != nil { + continue + } + if (in.Level != "" && run.Level != in.Level) || (in.Digest != "" && run.Digest != in.Digest) || (in.Module != "" && !readsModule(run, in.Module)) { + continue + } + t := run.Answer() + t.Key = key + r.packages(&t) + a.Runs = append(a.Runs, t) + } + if r.Fill != nil { + f, err := r.Fill() + if err != nil { + return seat.TestRunsAnswer{}, fmt.Errorf("the fill of test-runs cannot be read: %w", err) + } + a.Fill = f + } + return a, nil +} + +// packages names t's packages from its output object, or says it gone (§725, A-466v). +func (r RunsRead) packages(t *seat.TestRun) { + if t.Output == "" || r.Object == nil { + return + } + data, err := r.Object(t.Output) + if errors.Is(err, ErrNoRun) { + t.Gone = Gone + return + } + var out RunOutput + if err == nil { + err = json.Unmarshal(data, &out) + } + if err != nil { + t.Gone = "the run's output is not read: " + err.Error() + return + } + t.Packages = out.Packages +} + +// readsModule says whether a reader of run is a step of module, its artifact `:`. +func readsModule(run Run, module string) bool { + for _, o := range run.Readers { + if strings.HasPrefix(o.Artifact, module+":") { + return true + } + } + return false +} + +// TestRunsDrop is the seat verb test-runs-drop (design 47 G §728; A-469g, A-469v), answered by the runner's process: the +// hand act recorded first, the run's ask dropped, each reader ended could not run naming the drop, then the run's output +// object and its key deleted. It answers the run deleted and the readers it ended. +func (v Verbs) TestRunsDrop(caller string, in seat.TestRunsDropInput) (seat.TestRunsDropAnswer, error) { + h := v.Hearer + if h.TestRuns == nil { + return seat.TestRunsDropAnswer{}, errors.New("the runner keeps no test runs") + } + r, _, err := h.TestRuns.Get(in.Key) + if errors.Is(err, ErrNoRun) { + return seat.TestRunsDropAnswer{}, fmt.Errorf("no test run %q in test-runs", in.Key) + } + if err != nil { + return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not read: %w", in.Key, err) + } + if err := v.record(seat.VerbTestRunsDrop, "dropped the test run "+in.Key, in.Why); err != nil { + return seat.TestRunsDropAnswer{}, err + } + why := fmt.Sprintf("the test run %s was dropped by %s: %s", in.Key, caller, in.Why) + var ended []seat.TestRunReader + if r.Ended == nil { + if h.DropRun != nil && r.Ask != "" { + if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil { + h.logf("the drop of ask %s of the dropped test run %s is not answered: %v", r.Ask, in.Key, err) + } + } + if err := h.EndRun(in.Key, seat.TestRun{Ask: r.Ask, Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil); err != nil { + return seat.TestRunsDropAnswer{}, err + } + for _, o := range r.Readers { + ended = append(ended, o.Of()) + } + } + if h.RunOutputs != nil { + // The run's object is named by its key (endRunOf); one the run named otherwise is deleted too. + for _, name := range []string{in.Key, r.Output} { + if name == "" { + continue + } + if err := h.RunOutputs.Delete(name); err != nil { + return seat.TestRunsDropAnswer{}, fmt.Errorf("the output object %s of the test run is not deleted, so its key is kept: %w", name, err) + } + } + } + // The key is deleted at the revision of the run dropped: a run a rerun asked under it since is not deleted with it. + for { + now, rev, err := h.TestRuns.Get(in.Key) + if errors.Is(err, ErrNoRun) { + break + } + if err != nil { + return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not read again, so it is not deleted: %w", in.Key, err) + } + if now.Ask != r.Ask { + return seat.TestRunsDropAnswer{}, fmt.Errorf("the key %s holds the run of ask %s, asked since the run of ask %s was dropped: it is not deleted", in.Key, now.Ask, r.Ask) + } + if err := h.TestRuns.Delete(in.Key, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not deleted: %w", in.Key, err) + } + break + } + if h.Clock != nil { + for _, k := range []string{BoundPrefix + in.Key, UntakenPrefix + in.Key} { + _ = h.Clock.Unbound(k) + } + } + gone := r.Answer() + gone.Key = in.Key + now := h.now() + if gone.Ended == nil { + gone.Ended, gone.Outcome, gone.Said = &now, seat.StepResultOutcomeCouldNotRun, why + } + return seat.TestRunsDropAnswer{Run: gone, Ended: ended}, nil +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go new file mode 100644 index 00000000..ad9470ad --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -0,0 +1,672 @@ +package pipeline + +import ( + "fmt" + "strings" + "testing" + "time" + + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" +) + +// clockMem is the bound keys of test-runs in memory, each with the time to live it was last written with, the ask it holds +// and its revision. +type clockMem struct { + runs testRunsMem + bound map[string]time.Duration + asks map[string]string + revs map[string]uint64 + rev uint64 + // before is called once at the next Bound, a write that comes between its caller's read and its write. + before func() +} + +func (c *clockMem) Bound(key, ask string, ttl time.Duration) (uint64, error) { + if b := c.before; b != nil { + c.before = nil + b() + } + if c.asks == nil { + c.asks, c.revs = map[string]string{}, map[string]uint64{} + } + c.rev++ + c.bound[key], c.asks[key], c.revs[key] = ttl, ask, c.rev + return c.rev, nil +} +func (c *clockMem) Bounded(key string) (string, uint64, bool, error) { + _, ok := c.bound[key] + return c.asks[key], c.revs[key], ok, nil +} +func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil } +func (c *clockMem) UnboundAt(key string, rev uint64) error { + if c.revs[key] == rev { + delete(c.bound, key) + } + return nil +} +func (c *clockMem) Has(key string) (bool, error) { _, ok := c.bound[key]; return ok, nil } +func (c *clockMem) Keys() ([]string, error) { + var out []string + for k := range c.runs { + out = append(out, k) + } + for k := range c.bound { + out = append(out, k) + } + return out, nil +} + +type objectsMem map[string][]byte + +func (o objectsMem) Put(name string, data []byte) error { o[name] = data; return nil } +func (o objectsMem) Get(name string) ([]byte, error) { + if d, ok := o[name]; ok { + return d, nil + } + return nil, ErrNoRun +} +func (o objectsMem) Delete(name string) error { delete(o, name); return nil } + +// runFixture is a starter and a hearer over one pipelines store and one test-runs, the module's program and tools testing +// one directory. +type runFixture struct { + store memStore + runs testRunsMem + clock *clockMem + objects objectsMem + asks *stepAsks + drops []string + said []seat.StepEvent + logged []string + fill int64 + s *Starter + h *Hearer +} + +func newRunFixture() *runFixture { + f := &runFixture{store: memStore{}, runs: testRunsMem{}, objects: objectsMem{}, asks: &stepAsks{}} + f.clock = &clockMem{runs: f.runs, bound: map[string]time.Duration{}} + reg := catalog.Registration{Module: "app", Branches: []string{"main"}} + say := func(_ string, body any) error { + if e, ok := body.(seat.StepEvent); ok { + f.said = append(f.said, e) + } + return nil + } + f.h = &Hearer{Store: f.store, Say: say, TestRuns: f.runs, Clock: f.clock, RunOutputs: f.objects, + Logf: func(format string, a ...any) { f.logged = append(f.logged, fmt.Sprintf(format, a...)) }, + DropRun: func(seatName, node, ask, why string) error { + f.drops = append(f.drops, seatName+" "+node+" "+ask) + return nil + }} + f.s = &Starter{Pipelines: memCreated{f.store}, Store: f.store, TestRuns: f.runs, Clock: f.clock, Say: say, Steps: f.asks, + Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }, EndReaders: f.h.EndReaders, + RunsFill: func() (seat.Fill, error) { return seat.Fill{Store: "test-runs", Bytes: f.fill}, nil }, Logf: f.h.Logf} + return f +} + +func (f *runFixture) start(t *testing.T, source, revision int64) seat.Pipeline { + t.Helper() + entries := map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, + } + if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Module: "app", Commit: "c1", Fp: "src2:aa", Revision: revision, Entries: entries}); !ok || err != nil { + t.Fatalf("start = %v, %v", ok, err) + } + p, _, _ := f.store.Get(Key("main", source, "app")) + return p +} + +func (f *runFixture) pipeline(p seat.Pipeline) seat.Pipeline { + q, _, _ := f.store.Get(p.Pipeline) + return q +} + +// The test runs' clocks and cap (design 47 G §725; A-466n, A-466p, A-466s, A-468b, A-469d, A-470, A-470a): a run's untaken +// bound is written at its start, its clock at ask-taken and again at each step-progress; a clock's expiry ends the run and +// every reader could not run and drops its ask, a later step-ended ignored and journaled; an untaken bound of a taken run +// ends nothing; a run whose bound passed while the runner was away ends at its start; at the cap no run is asked; an ended +// run is aged with its output object in the step log's order. +func TestATestRunHasOneClockACapAndAnAge(t *testing.T) { + t.Run("clock", func(t *testing.T) { + f := newRunFixture() + f.h.Runs = RunSettings{StepBound: 30 * time.Minute, Untaken: 24 * time.Hour} + f.s.Runs = f.h.Runs + p := f.start(t, 4, 9) + unit := stepNamed(p, "app:program", "unit") + run := unit.Shared[0].Run + if f.clock.bound[UntakenPrefix+run] != 24*time.Hour || len(f.clock.bound) != 3 { + t.Fatalf("bounds at the start = %v, want the three runs' untaken bounds of 24h", f.clock.bound) + } + // A-469b: the run's steps wait for a build node until the holder's ask-taken, then run, each said started again. + if unit.Outcome != seat.StepResultOutcomeWaitingForABuildNode || stepNamed(p, "app:tools", "unit").Outcome != seat.StepResultOutcomeWaitingForABuildNode { + t.Fatalf("the unit steps before ask-taken = %s, %s", unit.Outcome, stepNamed(p, "app:tools", "unit").Outcome) + } + n := len(f.said) + if err := f.h.RunTaken(unit.Ask, "ace"); err != nil { + t.Fatal(err) + } + for _, art := range []string{"app:program", "app:tools"} { + if s := stepNamed(f.pipeline(p), art, "unit"); s.Outcome != seat.StepResultOutcomeRunning { + t.Errorf("the %s unit step after ask-taken is %s, want running", art, s.Outcome) + } + } + if len(f.said)-n != 2 || f.said[n].Result.Outcome != seat.StepResultOutcomeRunning { + t.Errorf("ask-taken said %d step events, want the two unit steps started running", len(f.said)-n) + } + if r, _, _ := f.runs.Get(run); r.Node != "ace" || f.clock.bound[BoundPrefix+run] != 30*time.Minute { + t.Fatalf("after ask-taken the run = %+v, bounds %v; want its node and its clock of 30m", r, f.clock.bound) + } + if _, ok := f.clock.bound[UntakenPrefix+run]; ok { + t.Error("the untaken bound of a taken run is kept") + } + // An untaken bound of a taken run ends nothing; progress begins the clock again. + if err := f.h.RunExpired(UntakenPrefix + run); err != nil { + t.Fatal(err) + } + delete(f.clock.bound, BoundPrefix+run) + if err := f.h.RunProgress(unit.Ask); err != nil || f.clock.bound[BoundPrefix+run] != 30*time.Minute { + t.Fatalf("progress = %v, bounds %v: want the clock begun again", err, f.clock.bound) + } + if r, _, _ := f.runs.Get(run); r.Ended != nil { + t.Fatal("a run is ended by its untaken bound after it was taken") + } + // An expiry heard once the bound is armed again ends nothing (A-470o); the bound holds the run's ask (A-470n). + if err := f.h.RunExpired(BoundPrefix + run); err != nil { + t.Fatal(err) + } + if r, _, _ := f.runs.Get(run); r.Ended != nil || f.clock.asks[BoundPrefix+run] != unit.Ask { + t.Fatalf("an expiry of a bound armed again ended the run %+v, or the bound holds ask %q", r, f.clock.asks[BoundPrefix+run]) + } + delete(f.clock.bound, BoundPrefix+run) // the bus's expiry deletes it + if err := f.h.RunExpired(BoundPrefix + run); err != nil { + t.Fatal(err) + } + r, _, _ := f.runs.Get(run) + if r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, unit.Ask) || !strings.Contains(r.Said, "30m") { + t.Errorf("the run whose clock ended = %+v, want could not run naming its ask and the bound", r) + } + if len(f.drops) != 1 || f.drops[0] != "node-build-go ace "+unit.Ask { + t.Errorf("drops = %v, want the run's ask dropped of ace", f.drops) + } + q := f.pipeline(p) + for _, art := range []string{"app:program", "app:tools"} { + if s := stepNamed(q, art, "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != r.Said { + t.Errorf("the %s unit step = %+v, want could not run with the run", art, s) + } + } + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask, + Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "passed", Ended: &ended}); err != nil { + t.Fatal(err) + } + if s := stepNamed(f.pipeline(p), "app:program", "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun { + t.Errorf("a step-ended after the run ended moved the step to %s", s.Outcome) + } + if !strings.Contains(strings.Join(f.logged, "\n"), "the step-ended of ask "+unit.Ask+" is ignored: its test run "+run) { + t.Errorf("the ignored step-ended is not journaled: %q", f.logged) + } + // An untaken run whose bound passed while the runner was away ends at the start. + lint := stepNamed(p, "app:program", "lint").Shared[0].Run + delete(f.clock.bound, UntakenPrefix+lint) + if n, err := f.h.TakeUp(); n != 1 || err != nil { + t.Fatalf("take-up ended %d, %v; want the lint run", n, err) + } + if r, _, _ := f.runs.Get(lint); r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "run-untaken-bound") { + t.Errorf("the lint run taken up = %+v", r) + } + }) + t.Run("cap", func(t *testing.T) { + f := newRunFixture() + f.s.Runs.CapBytes, f.fill = 1<<20, 1<<20 + p := f.start(t, 4, 9) + for _, a := range f.asks.ids { + if a == stepNamed(p, "app:program", "unit").Ask { + t.Error("a unit run is asked at the cap") + } + } + q := f.pipeline(p) + for _, art := range []string{"app:program", "app:tools"} { + if s := stepNamed(q, art, "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(s.Why, "1048576 of its 1048576 bytes") { + t.Errorf("the %s unit step at the cap = %+v, want could not run naming the fill", art, s) + } + } + }) + t.Run("age", func(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + unit := stepNamed(p, "app:program", "unit") + run := unit.Shared[0].Run + _ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed, Output: "out-1"}, nil) + f.objects["out-1"] = []byte("ok") + now := agedNow + a := &Ager{Runs: f.runs, RunOutputs: f.objects.Delete, RunsKept: 7 * 24 * time.Hour, Pipelines: agedDeleter{f.store}, + Aged: agedState{}, Now: func() time.Time { return now }, + Log: func(from uint64, limit int) ([]LoggedEntry, error) { + if from > 1 { + return nil, nil + } + return []LoggedEntry{{Sequence: 1, Entry: seat.LogEntry{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools", + What: seat.LogEntryWhatEnded, At: agedNow.Add(-6 * 24 * time.Hour)}}}, nil + }} + a.Age() + if _, _, err := f.runs.Get(run); err != nil { + t.Fatal("a run is aged before test-runs-kept passed") + } + now = now.Add(2 * 24 * time.Hour) + a.Age() + if _, _, err := f.runs.Get(run); err != ErrNoRun || len(f.objects) != 0 { + t.Errorf("the run past test-runs-kept is %v, objects %v; want both deleted", err, f.objects) + } + }) +} + +// agedDeleter is a memStore with the delete the ager makes. +type agedDeleter struct{ memStore } + +func (d agedDeleter) Delete(key string) error { delete(d.memStore, key); return nil } + +// How a test run ends and is said (design 47 G §726; A-466i, A-468d, A-469m, A-469z): the run's ask's end ends every step +// reading it with its outcome, counts and shared, its output kept in test-runs-objects; a pipeline counts the run's tests +// once; a step superseded leaves the run to its other readers, and the last to leave drops its ask and ends it could not +// run, no reader left; a composition step's end keeps the runs mesh-wants names. +func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) { + f := newRunFixture() + a := f.start(t, 4, 9) + b := f.start(t, 6, 9) // the same candidate: it reads a's runs + unit := stepNamed(a, "app:program", "unit") + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask, + Pipeline: a.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", + Said: "TestX failed", Output: "--- FAIL: TestX", Tests: &nodebuildgo.TestCounts{Unit: 5}, Ended: &ended}); err != nil { + t.Fatal(err) + } + for _, p := range []seat.Pipeline{f.pipeline(a), f.pipeline(b)} { + for _, art := range []string{"app:program", "app:tools"} { + s := stepNamed(p, art, "unit") + if s.Outcome != seat.StepResultOutcomeFailed || s.Why != "TestX failed" || s.Tests == nil || s.Tests.Unit != 5 || + len(s.Shared) != 1 || s.Shared[0].Run != unit.Shared[0].Run || s.Shared[0].Ask != unit.Ask { + t.Errorf("%s's %s unit step = %+v, want failed with the run's result and shared", p.Pipeline, art, s) + } + } + if c := testCounts(p.Steps); c == nil || c.Unit != 5 { + t.Errorf("%s counts %+v, want the run's 5 unit tests once", p.Pipeline, c) + } + } + if r, _, _ := f.runs.Get(unit.Shared[0].Run); r.Outcome != "failed" || r.Output != unit.Shared[0].Run || !strings.Contains(string(f.objects[r.Output]), `"output":"--- FAIL: TestX"`) { + t.Errorf("the ended run = %+v, objects %v; want its output kept under its key", r, f.objects) + } + saidReaders := 0 + for _, e := range f.said { + if e.Result.Step == "unit" && e.Result.Outcome == "failed" { + saidReaders++ + } + } + if saidReaders != 4 { + t.Errorf("said %d unit step-ended, want one per reading step", saidReaders) + } + // a, superseded, leaves the lint run to b's readers; b, superseded after, leaves none: the ask is dropped and the run ends. + lint := stepNamed(a, "app:program", "lint") + if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.5.app", Branch: "main", Source: 5, Module: "app", Commit: "c2", Fingerprint: "src2:bb"}, nil); err != nil { + t.Fatal(err) + } + if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Ended != nil || len(r.Readers) != 2 || len(f.drops) != 0 { + t.Fatalf("after a left the lint run = %+v, drops %v; want it running for b's two readers", r, f.drops) + } + if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.7.app", Branch: "main", Source: 7, Module: "app", Commit: "c3", Fingerprint: "src2:cc"}, nil); err != nil { + t.Fatal(err) + } + if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Outcome != seat.StepResultOutcomeCouldNotRun || r.Said != NoReaderLeft || len(f.drops) != 2 || f.drops[0] != "node-build-go "+lint.Ask { + t.Errorf("after the last reader left the lint run = %+v, drops %v; want it ended, no reader left, its ask and the integration run's dropped", r, f.drops) + } + // A composition step's end keeps the runs mesh-wants names, as they came (A-468d). + c := newRunFixture() + p := c.start(t, 4, 9) + comp := stepNamed(p, "", "composition") + bs := stepSeatOf(t, "mesh-wants") + raw := []byte(`{"id":"` + comp.Ask + `","pipeline":"main.4.app","step":"composition","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"passed",` + + `"shared":[{"machine":"ace","run":"main.9.ace","ask":"r1"}]}`) + ev, err := bs.Decode(raw) + if err != nil { + t.Fatal(err) + } + if _, err := c.h.OnStepEndedShared(bs, bs.Prefix+"n1", ev, bs.Shared(raw)); err != nil { + t.Fatal(err) + } + if s := stepNamed(c.pipeline(p), "", "composition"); len(s.Shared) != 1 || s.Shared[0] != (seat.Shared{Machine: "ace", Run: "main.9.ace", Ask: "r1"}) { + t.Errorf("the composition step's shared = %+v, want mesh-wants' as it came", s.Shared) + } +} + +// Asking a test run again moves every reader (design 47 G §727; A-469l, A-470d, A-470e): a rerun of one artifact's failed +// step starts a new run under the same key, carrying the former readers, the one that did not pass marked; the new run's +// passing end moves the other artifact's step to passed, marked a rerun by the caller and naming the new ask, and the +// pipeline says pipeline-ended again; a step a person accepted does not move. +func TestARerunOfATestRunMovesEveryReaderThatDidNotPass(t *testing.T) { + // takeUp: the rerun's run is ended by the restart's take-up, its bounds gone while the runner was away; a carried reader + // with rerun false, its step passed already, says no step-ended again (A-470l). + for _, c := range []struct{ accepted, takeUp bool }{{false, false}, {true, false}, {false, true}} { + accepted := c.accepted + f := newRunFixture() + reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required", + Integration: "bypass", Composition: "bypass", Review: "bypass", Approval: "bypass"}}} + f.s.Registration = func(string) (catalog.Registration, bool) { return reg, true } + p := f.start(t, 4, 9) + e := f.store[p.Pipeline] + e.p.Steps[0].Outcome = seat.StepResultOutcomePassed // the build + f.store[p.Pipeline] = e + unit := stepNamed(p, "app:program", "unit") + end := func(ask string, outcome string, rerun bool) { + t.Helper() + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: ask, + Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: outcome, + Rerun: rerun, By: map[bool]string{true: "op@shanks"}[rerun], Ended: &ended}); err != nil { + t.Fatal(err) + } + } + end(unit.Ask, "failed", false) + if q := f.pipeline(p); q.Outcome != seat.PipelineOutcomeFailed || q.EndCount != 1 || stepNamed(q, "app:tools", "unit").Outcome != "failed" { + t.Fatalf("after the failing run the pipeline = %s (%d), tools %s", q.Outcome, q.EndCount, stepNamed(q, "app:tools", "unit").Outcome) + } + if accepted { + e := f.store[p.Pipeline] + i := stepIndex(e.p, "unit", "app:program") + e.p.Steps[i].Outcome, e.p.Steps[i].Accepted = seat.StepResultOutcomeAccepted, &seat.Accepted{By: "op@shanks", Why: "known"} + f.store[p.Pipeline] = e + } + v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) { + return Half{Entries: map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true + }, Acts: &actLog{}} + a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools", Why: "flaky"}) + if err != nil { + t.Fatal(err) + } + again := a.Result.Ask + if c.takeUp { + run := unit.Shared[0].Run + r, rev, _ := f.runs.Get(run) + r.Readers[0].Rerun, r.Readers[0].By = false, "" // the program's step passed already + if _, err := f.runs.Update(run, r, rev); err != nil { + t.Fatal(err) + } + e := f.store[p.Pipeline] + e.p.Steps[stepIndex(e.p, "unit", "app:program")].Outcome = seat.StepResultOutcomePassed + f.store[p.Pipeline] = e + for k := range f.clock.bound { + delete(f.clock.bound, k) + } + said := len(f.said) + if n, err := f.h.TakeUp(); err != nil || n != 1 { + t.Fatalf("take-up = %d, %v", n, err) + } + for _, ev := range f.said[said:] { + if ev.Result.Artifact == "app:program" { + t.Errorf("a carried reader with rerun false said %s again: %+v", ev.Result.Outcome, ev) + } + } + q := f.pipeline(p) + if stepNamed(q, "app:program", "unit").Outcome != seat.StepResultOutcomePassed || stepNamed(q, "app:tools", "unit").Outcome != seat.StepResultOutcomeCouldNotRun { + t.Errorf("after the take-up program %s, tools %s; want passed kept, tools could not run", + stepNamed(q, "app:program", "unit").Outcome, stepNamed(q, "app:tools", "unit").Outcome) + } + continue + } + r, _, _ := f.runs.Get(unit.Shared[0].Run) + if again == "" || again == unit.Ask || r.Ask != again || r.Ended != nil || len(r.Readers) != 2 || r.Readers[0].Rerun == accepted || (!accepted && r.Readers[0].By != "op@shanks") { + t.Fatalf("the rerun's run = %+v (ask %s), want a new run under the same key carrying the program, marked unless accepted", r, again) + } + end(again, "passed", true) + q := f.pipeline(p) + prog := stepNamed(q, "app:program", "unit") + if accepted { + if prog.Outcome != seat.StepResultOutcomeAccepted { + t.Errorf("an accepted step moved to %s", prog.Outcome) + } + continue + } + if prog.Outcome != seat.StepResultOutcomePassed || !prog.Rerun || prog.By != "op@shanks" || prog.Shared[0].Ask != again { + t.Errorf("the program's step = %+v, want passed, a rerun by the caller, naming the new ask", prog) + } + if q.Outcome != seat.PipelineOutcomePassed || q.EndCount != 2 { + t.Errorf("the pipeline = %s (%d), want passed at its second end", q.Outcome, q.EndCount) + } + } +} + +// The test runs' tools (design 47 G §728, §733 unit; A-466v, A-469g, A-469u, A-469v): test-runs answers each run with its +// packages read from its output object, and a run whose object is gone without them, saying so; test-runs-drop records the +// hand act first, ends the readers could not run naming the drop, and deletes the key and its object; with no hand act +// recorded it does nothing. +func TestTheTestRunsAreReadAndDroppedByTheirVerbs(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + unit, lint := stepNamed(p, "app:program", "unit"), stepNamed(p, "app:program", "lint") + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask, + Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "passed", + Kept: []nodebuildgo.KeptResult{{Package: "app/x", Digest: "d1", Level: "unit"}}, Ended: &ended}); err != nil { + t.Fatal(err) + } + pipes := memKept{} + for k, e := range f.runs { + put(t, pipes, k, e.r) + } + // The fill is said against the cap setting that refuses, never the bucket's 64 MiB ceiling (A-470g, A-470i). + f.s.Runs.CapBytes, f.fill = 1<<20, 9 + bucket := f.s.RunsFill + f.s.RunsFill = func() (seat.Fill, error) { fl, err := bucket(); fl.Cap = 64 << 20; return fl, err } + read := RunsRead{Runs: pipes, Object: f.objects.Get, Fill: f.s.TestRunsFill} + a, err := read.TestRuns(seat.TestRunsInput{Level: "unit", Module: "app"}) + if err != nil || len(a.Runs) != 1 || len(a.Runs[0].Packages) != 1 || a.Runs[0].Packages[0] != (seat.TestRunPackage{Package: "app/x", Digest: "d1"}) || + a.Runs[0].Key != unit.Shared[0].Run || len(a.Runs[0].Readers) != 2 || a.Fill.Cap != 1<<20 || a.Fill.Bytes != 9 { + t.Fatalf("test-runs = %+v, %v; want the unit run with its package and readers, and the fill against the cap setting", a, err) + } + delete(f.objects, unit.Shared[0].Run) // gone, whatever step-outputs' age: the run answers without its packages + if a, _ := read.TestRuns(seat.TestRunsInput{Level: "unit"}); len(a.Runs) != 1 || a.Runs[0].Gone != Gone || len(a.Runs[0].Packages) != 0 { + t.Errorf("a run whose output is gone = %+v", a.Runs) + } + acts := &actLog{} + v := Verbs{Hearer: f.h, Starter: f.s, Acts: acts} + if _, err := (Verbs{Hearer: f.h, Starter: f.s}).TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: lint.Shared[0].Run, Why: "stuck"}); err == nil { + t.Fatal("a drop with no hand act recorded was done") + } + f.objects[lint.Shared[0].Run] = []byte("{}") + d, err := v.TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: lint.Shared[0].Run, Why: "stuck"}) + if err != nil || len(*acts) != 1 || len(d.Ended) != 2 || d.Run.Key != lint.Shared[0].Run || len(f.drops) != 1 { + t.Fatalf("drop = %+v, %v; acts %v, drops %v", d, err, *acts, f.drops) + } + if _, _, err := f.runs.Get(lint.Shared[0].Run); err != ErrNoRun || f.objects[lint.Shared[0].Run] != nil { + t.Errorf("after the drop the key is %v and the object %q, want both gone", err, f.objects[lint.Shared[0].Run]) + } + if s := stepNamed(f.pipeline(p), "app:tools", "lint"); s.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(s.Why, "dropped by op@shanks") { + t.Errorf("a reader of the dropped run = %+v", s) + } +} + +// racingRuns is test-runs with a write coming between a caller's read and its next update or delete: each hook is called +// once, just before that write. +type racingRuns struct { + testRunsMem + beforeUpdate, beforeDelete func() +} + +func (r *racingRuns) Update(key string, v Run, rev uint64) (uint64, error) { + if b := r.beforeUpdate; b != nil { + r.beforeUpdate = nil + b() + } + return r.testRunsMem.Update(key, v, rev) +} + +func (r *racingRuns) Delete(key string, rev uint64) error { + if b := r.beforeDelete; b != nil { + r.beforeDelete = nil + b() + } + return r.testRunsMem.Delete(key, rev) +} + +// No write of test-runs or its bound keys puts back, ends or deletes a run from a read made before its latest change (design +// 47 G §725, §726, §727, §728; the kind test 109 met): each write is a compare-and-set at the revision read, retried from the +// latest read, and a judgement of a run ends only the run under the ask it judged. +func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) { + asked := func(f *runFixture, run string) { + t.Helper() + r, rev, _ := f.runs.Get(run) + r.TestRun = seat.TestRun{Level: r.Level, Digest: r.Digest, Seat: r.Seat, Ask: "a2"} + if _, err := f.runs.Update(run, r, rev); err != nil { + t.Fatal(err) + } + } + for _, c := range []struct { + name string + race func(t *testing.T, f *runFixture, p seat.Pipeline, run string) + }{ + {"an expiry does not end a run asked again while its ask was dropped", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { + unit := stepNamed(p, "app:program", "unit") + _ = f.h.RunTaken(unit.Ask, "ace") + f.h.DropRun = func(string, string, string, string) error { asked(f, run); return nil } + delete(f.clock.bound, BoundPrefix+run) // the bus's expiry deletes it + _ = f.h.RunExpired(BoundPrefix + run) + }}, + {"the last reader to leave ends the run in its own write, so a step joining then asks its own", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { + f.h.DropRun = func(string, string, string, string) error { + _, _, _ = f.s.claimRun(Asked{Run: run, ID: "a2", Level: "unit", Seat: "node-build-go", + Readers: []seat.TestRunReader{{Pipeline: "main.5.app", Artifact: "app:program"}}}) + return nil + } + for _, a := range []string{"app:program", "app:tools"} { + f.h.leaveRun(p.Pipeline, stepNamed(p, a, "unit"), "cancelled") + } + }}, + {"the age does not delete a run asked again since it was read", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { + _ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed}, nil) + runs := &racingRuns{testRunsMem: f.runs, beforeDelete: func() { asked(f, run) }} + a := &Ager{Runs: runs, RunsKept: time.Hour, Pipelines: agedDeleter{f.store}, Aged: agedState{}, Now: func() time.Time { return agedNow }, + Log: func(from uint64, limit int) ([]LoggedEntry, error) { + if from > 1 { + return nil, nil + } + return []LoggedEntry{{Sequence: 1, Entry: seat.LogEntry{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools", + What: seat.LogEntryWhatEnded, At: agedNow.Add(-2 * time.Hour)}}}, nil + }} + a.Age() + }}, + {"test-runs-drop does not delete a run asked again under its key", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { + f.h.TestRuns = &racingRuns{testRunsMem: f.runs, beforeDelete: func() { asked(f, run) }} + v := Verbs{Hearer: f.h, Starter: f.s, Acts: &actLog{}} + if _, err := v.TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: run, Why: "stuck"}); err == nil { + t.Error("a drop whose key holds a run asked since answers no error") + } + }}, + {"a step-progress heard as the run ends leaves no clock", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { + unit := stepNamed(p, "app:program", "unit") + _ = f.h.RunTaken(unit.Ask, "ace") + f.clock.before = func() { _ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed}, nil) } + _ = f.h.RunProgress(unit.Ask) + if _, ok := f.clock.bound[BoundPrefix+run]; ok { + t.Error("a clock written from a read before the run's end is kept") + } + asked(f, run) // the key's run is asked again, so the check below holds + }}, + {"a clock of a run not taken ends nothing", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { + asked(f, run) + _ = f.h.RunExpired(BoundPrefix + run) + }}, + } { + t.Run(c.name, func(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + run := stepNamed(p, "app:program", "unit").Shared[0].Run + c.race(t, f, p, run) + if r, _, err := f.runs.Get(run); err != nil || r.Ask != "a2" || r.Ended != nil { + t.Errorf("the key's run = %+v, %v; want the run asked again under a2, running", r, err) + } + }) + } + // A late signal of a replaced run arms no bound and is said in the journal (A-470n). + t.Run("a late ask-taken or step-progress of a replaced run moves nothing", func(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + unit := stepNamed(p, "app:program", "unit") + run := unit.Shared[0].Run + asked(f, run) + delete(f.clock.bound, UntakenPrefix+run) + _ = f.h.RunTaken(unit.Ask, "ace") + _ = f.h.RunProgress(unit.Ask) + late := 0 + for _, l := range f.logged { + if strings.Contains(l, unit.Ask) && strings.Contains(l, "moves nothing") { + late++ + } + } + if _, ok := f.clock.bound[BoundPrefix+run]; ok || late != 2 { + t.Errorf("bounds %v, journal %q; want no clock and both late signals journaled", f.clock.bound, f.logged) + } + }) + // A run's end is written as its step-ended is heard, before the finding the step says (A-470o). + t.Run("the run ends before the step's finding is written", func(t *testing.T) { + f := newRunFixture() + reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required", + Integration: "bypass", Composition: "bypass", Review: "bypass", Approval: "bypass"}}} + f.s.Registration = func(string) (catalog.Registration, bool) { return reg, true } + p := f.start(t, 4, 9) + unit := stepNamed(p, "app:program", "unit") + run := unit.Shared[0].Run + endedFirst := false + f.h.Failures = &Failures{Findings: newHeld(), Author: func(string, string) (string, error) { + r, _, _ := f.runs.Get(run) + endedFirst = r.Ended != nil + return "alice", nil + }} + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask, + Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", Ended: &ended}); err != nil { + t.Fatal(err) + } + if !endedFirst { + t.Error("the step's finding was written before its run's end") + } + }) + // A rerun whose read of the key is changed before its write (a step joining the ended run) writes from the latest read, + // carrying that reader, never answering the change as an error nor putting back the earlier readers. + t.Run("a rerun writes its run from the latest read", func(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + unit := stepNamed(p, "app:program", "unit") + run := unit.Shared[0].Run + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask, + Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", Ended: &ended}); err != nil { + t.Fatal(err) + } + f.h.TestRuns = &racingRuns{testRunsMem: f.runs, beforeUpdate: func() { + r, rev, _ := f.runs.Get(run) + r.Readers = append(r.Readers, RunReader{Pipeline: "main.9.app", Artifact: "app:program"}) + _, _ = f.runs.Update(run, r, rev) + }} + v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) { + return Half{Entries: map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true + }, Acts: &actLog{}} + a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:program", Why: "fixed"}) + if err != nil { + t.Fatalf("a rerun whose read was changed before its write: %v", err) + } + r, _, _ := f.runs.Get(run) + joined := false + for _, o := range r.Readers { + joined = joined || o.Pipeline == "main.9.app" + } + if r.Ask != a.Result.Ask || r.Ended != nil || !joined || len(r.Readers) != 3 { + t.Errorf("the rerun's run = %+v; want the new ask running, carrying the reader that joined", r) + } + }) +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index b12a19f7..5ca58405 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -27,8 +27,8 @@ type Verbs struct { Candidate func(group, branch string, revision int64) (Half, error) // Acts records a hand act before the verb does it (rerun, accept, cancel); a nil Acts refuses the verb, nothing recorded. Acts views.HandActs - // Drop drops the bound of an ask a cancelled pipeline no longer waits on. - Drop func(ask string) error + // Runs answers the read verb test-runs, answered by the process that holds test-runs-cap-mib (A-470i). + Runs RunsRead } func (v Verbs) now() time.Time { return v.Hearer.now() } @@ -185,8 +185,8 @@ func (v Verbs) Accept(caller string, in seat.AcceptInput) (seat.AcceptAnswer, er return seat.AcceptAnswer{Result: p.Steps[i]}, nil } -// Cancel ends a running pipeline `cancelled`, each step not ended cancelled and the bound of each ask dropped, recorded in -// the hand-act log first (§394); it says pipeline-ended. +// Cancel ends a running pipeline `cancelled`, each step not ended cancelled and each step of a test run leaving it, recorded +// in the hand-act log first (§394, §726); it says pipeline-ended. func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, error) { h := v.Hearer recorded := false @@ -205,14 +205,14 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er recorded = true } now := h.now() - var drop []string + var leave []seat.StepResult for i := range p.Steps { s := &p.Steps[i] - if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting { + if !open(*s) && s.Outcome != seat.StepResultOutcomeWaiting { continue } - if s.Ask != "" { - drop = append(drop, s.Ask) + if sharedRun(*s) != "" && s.Seat != wants.Seat { + leave = append(leave, *s) // it leaves its test run, dropped only with the last reader (§726) } s.Outcome, s.By, s.Why, s.Ended = seat.StepResultOutcomeCancelled, caller, in.Why, &now } @@ -223,12 +223,8 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er } else if err != nil { return seat.CancelAnswer{}, fmt.Errorf("pipeline %s cannot be written: %w", in.Pipeline, err) } - if v.Drop != nil { - for _, ask := range drop { - if err := v.Drop(ask); err != nil { - h.logf("the bound of ask %s of the cancelled %s is not dropped: %v", ask, in.Pipeline, err) - } - } + for _, s := range leave { + h.leaveRun(in.Pipeline, s, "the pipeline "+in.Pipeline+" is cancelled: "+in.Why) } return seat.CancelAnswer{Pipeline: p}, h.said(p, false, true) } @@ -242,6 +238,11 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error if in.Step == seat.StepResultStepApproval { return seat.RerunAnswer{}, errors.New("an approval is asked again by ask-again, not rerun") } + if p, _, err := v.Hearer.Store.Get(in.Pipeline); err == nil { + if i, err := stepOf(p, in.Step, in.Artifact); err == nil && v.Hearer.TestRuns != nil && sharedRun(p.Steps[i]) != "" && BoundSeat(p.Steps[i].Seat) { + return v.rerunTest(caller, in, p.Steps[i]) + } + } var ask *Asked p, _, err := v.change(in.Pipeline, seat.EventStepStarted, func() error { return v.record(seat.VerbRerun, fmt.Sprintf("asked the %s step of %s again", in.Step, in.Pipeline), in.Why) @@ -289,7 +290,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error v.Hearer.pin(p) // a pipeline that runs again keeps its revision again (A-449) if ask != nil { v.Starter.carry(in.Pipeline, []Asked{*ask}) - if err := askAll(v.Starter.Steps, v.Starter.Arm, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil { + if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil { return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err) } } @@ -301,6 +302,173 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &r}, nil } +// rerunTest asks a test step of a test run again (design 47 G §727, A-469l): it joins its key's run while that runs, reads one +// that started after the step's last end, or else starts a new run under a new ask, replacing the ended key by compare-and-set +// and carrying its readers into the new value, so the new run's end moves every former reader that did not pass. +func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) (seat.RerunAnswer, error) { + h := v.Hearer + run := sharedRun(st) + if open(st) { + return seat.RerunAnswer{}, fmt.Errorf("the %s step of %s is running: it is not asked again", in.Step, in.Pipeline) + } + r, rev, err := h.TestRuns.Get(run) + if err != nil && !errors.Is(err, ErrNoRun) { + return seat.RerunAnswer{}, fmt.Errorf("the test run %s is not read: %w", run, err) + } + exists := err == nil + reader := seat.TestRunReader{Pipeline: in.Pipeline, Artifact: st.Artifact} + read := exists && r.Ended != nil && r.Outcome != seat.StepResultOutcomeCouldNotRun && st.Ended != nil && r.Started != nil && r.Started.After(*st.Ended) + join := exists && r.Ended == nil + var ask *Asked + if !join && !read { + p, _, err := h.Store.Get(in.Pipeline) + if err != nil { + return seat.RerunAnswer{}, err + } + if ask, err = v.askOf(p, st, caller, h.now()); err != nil { + return seat.RerunAnswer{}, err + } + } + var asker *seat.SharedAskedBy + if join || read { + if asker, err = askerOf(h.Store, r.Ask); err != nil { + return seat.RerunAnswer{}, err + } + if asker != nil && *asker == (seat.SharedAskedBy{Pipeline: reader.Pipeline, Artifact: reader.Artifact}) { + asker = nil + } + } + event := seat.EventStepStarted + if read { + event = seat.EventStepEnded + } + p, _, err := v.change(in.Pipeline, event, func() error { + return v.record(seat.VerbRerun, fmt.Sprintf("asked the %s step of %s again", in.Step, in.Pipeline), in.Why) + }, func(p *seat.Pipeline, now time.Time) (int, error) { + switch p.Outcome { + case seat.PipelineOutcomeCancelled, seat.PipelineOutcomeSuperseded: + return -1, fmt.Errorf("the pipeline %s is %s: a step of it is not asked again", p.Pipeline, p.Outcome) + } + i, err := stepOf(*p, in.Step, in.Artifact) + if err != nil { + return -1, err + } + s := &p.Steps[i] + *s = seat.StepResult{Step: s.Step, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint, Seat: s.Seat, + Outcome: seat.StepResultOutcomeRunning, By: caller, Rerun: true, Started: &now} + switch { + case ask != nil: + s.Ask, s.Shared = ask.ID, []seat.Shared{{Run: run, Ask: ask.ID}} + s.Outcome, s.On = seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode // until ask-taken (A-469b) + case read: + s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}} + s.Outcome, s.Why, s.Node, s.Ended = r.Outcome, r.Said, r.Node, &now + if r.Tests != nil { + t := *r.Tests + s.Tests = &t + } + default: + s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}} + if r.Node == "" { + s.Outcome, s.On = seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode + } + } + if p.Outcome != seat.PipelineOutcomeRunning { + p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil + } + for j := range p.Steps { + if p.Steps[j].Step == seat.StepResultStepApproval && p.Steps[j].Outcome == seat.StepResultOutcomeNotAsked { + p.Steps[j].Outcome, p.Steps[j].Why = seat.StepResultOutcomeWaiting, "" + } + } + return i, nil + }) + if err != nil { + return seat.RerunAnswer{}, err + } + h.pin(p) + switch { + case join || read: + v.Starter.joinRun(run, reader) + default: + now := h.now() + a := *ask + a.Run = run + // The new run is written from the latest read of the key: a write between (a reader leaving, a join) is read again and + // its readers carried, never put back; a run another step started since is said, not replaced. + for { + value := Run{TestRun: seat.TestRun{Level: st.Step, Digest: r.Digest, Seat: st.Seat, Ask: ask.ID, Started: &now}} + // Every former reader is carried, each that did not pass marked to move with the new run by this caller (A-470d). + for _, o := range r.Readers { + if o.Of() == reader { + continue + } + o.Rerun, o.By = !v.passedReader(o, run), "" + if o.Rerun { + o.By = caller + } + value.Readers = append(value.Readers, o) + } + addReaders(&value, []seat.TestRunReader{reader}) + err := v.Starter.started(a, func() error { + if !exists { + return h.TestRuns.Create(run, value) + } + _, err := h.TestRuns.Update(run, value, rev) + return err + }) + if errors.Is(err, ErrChanged) || errors.Is(err, ErrExists) { + got, gotRev, gerr := h.TestRuns.Get(run) + switch { + case errors.Is(gerr, ErrNoRun): + r, rev, exists = Run{TestRun: seat.TestRun{Digest: r.Digest}}, 0, false + continue + case gerr != nil: + err = gerr + case got.Ended == nil: + err = fmt.Errorf("the run of ask %s was asked under the key since it was read", got.Ask) + default: + r, rev, exists = got, gotRev, true + continue + } + } + if err != nil { + return seat.RerunAnswer{}, fmt.Errorf("the step is written running but its test run %s is not: %w", run, err) + } + break + } + v.Starter.carry(in.Pipeline, []Asked{*ask}) + if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil { + return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err) + } + } + i, err := stepOf(p, in.Step, in.Artifact) + if err != nil { + return seat.RerunAnswer{}, err + } + res := p.Steps[i] + return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &res}, nil +} + +// passedReader says whether reader's step of run passed, or a person accepted it (§30): a step that moves with no new run. +func (v Verbs) passedReader(reader RunReader, run string) bool { + p, _, err := v.Hearer.Store.Get(reader.Pipeline) + if err != nil { + return true // a pipeline gone has no step to move + } + for _, s := range p.Steps { + if s.Artifact == reader.Artifact && sharedRun(s) == run { + switch s.Outcome { + case seat.StepResultOutcomePassed, seat.StepResultOutcomeCarried, seat.StepResultOutcomeAccepted, + seat.StepResultOutcomeSuperseded, seat.StepResultOutcomeCancelled: + return true + } + return s.Accepted != nil + } + } + return true +} + // askOf is the ask to put on the serving seat for step s again: the one Place makes for it, under a new id (an ask put twice // under one id is kept once), saying it is a rerun and by whom. func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time.Time) (*Asked, error) { @@ -325,7 +493,7 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time } else { return nil, fmt.Errorf("the catalog half of %s is not held yet: the step is asked when it is", p.Branch) } - _, placed := Place(p, reg, half.Entries, now, v.Starter.Name) + _, placed := place(p, reg, half.Entries, now, v.Starter.Name, false) base := AskID(startOf(p, now), p.Pipeline, s.Artifact, s.Step) for _, a := range placed { if a.ID != base { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go index 6b6ed3df..bb355a07 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go @@ -35,8 +35,9 @@ func verbsFixture() (Verbs, memStore, *[]string, *stepAsks, *actLog, *[]string) asks, acts, dropped := &stepAsks{}, &actLog{}, &[]string{} s := &Starter{Steps: asks, Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }, Now: h.Now} - return Verbs{Hearer: h, Starter: s, View: func(string) (Half, bool) { return Half{Entries: entries}, true }, Acts: acts, - Drop: func(ask string) error { *dropped = append(*dropped, ask); return nil }}, store, said, asks, acts, dropped + h.DropRun = func(_, _, ask, _ string) error { *dropped = append(*dropped, ask); return nil } + return Verbs{Hearer: h, Starter: s, View: func(string) (Half, bool) { return Half{Entries: entries}, true }, Acts: acts}, + store, said, asks, acts, dropped } func stepNamed(p seat.Pipeline, artifact, step string) seat.StepResult { @@ -81,7 +82,8 @@ func TestReviewEndsThePlacedReviewStepWithItsProvenCaller(t *testing.T) { // accept takes only a failed optional step, writes its hand act first, and refuses without one; rerun asks the step again // under a new ask saying it is a rerun and its caller, reopens a failed pipeline and asks nothing of a refused one; cancel ends -// a running pipeline cancelled, each step cancelled and each ask's bound dropped (design 47 G §22, §30, §325, §394). +// a running pipeline cancelled, each step cancelled, no ask's bound left to drop (design 47 G §22, §30, §325, §394; §647 as +// A-466y amends it). func TestAcceptRerunAndCancelWriteByCompareAndSetAndRecordTheirHandAct(t *testing.T) { v, store, said, asks, acts, dropped := verbsFixture() end := func(step, outcome string) { @@ -134,7 +136,7 @@ func TestAcceptRerunAndCancelWriteByCompareAndSetAndRecordTheirHandAct(t *testin } // A cancel ends the pipeline cancelled and drops the bounds of the asks still held. c, err := v.Cancel("operator", seat.CancelInput{Pipeline: pkey, Why: "wrong branch"}) - if err != nil || c.Pipeline.Outcome != "cancelled" || c.Pipeline.EndCount != 1 || len(*dropped) == 0 { + if err != nil || c.Pipeline.Outcome != "cancelled" || c.Pipeline.EndCount != 1 || len(*dropped) != 0 { t.Fatalf("cancel = %+v, %v, dropped %v", c.Pipeline, err, *dropped) } if s := stepNamed(c.Pipeline, "app:program", "lint"); s.Outcome != "cancelled" || s.By != "operator" { diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 80d88e8a..9b3cb607 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -55,9 +55,12 @@ func TestManifest(t *testing.T) { {"claims the seat serving every verb of its definition", len(m.Claims) == 1 && m.Claims[0].Name == seat.Seat && equalSets(m.Claims[0].Serves, seat.Verbs())}, // §27, §85, §330, §355, §77: the old state sections. - {"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "toolchains", "findings", "first-views", "consumers"})}, + {"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "test-runs", "toolchains", "findings", "first-views", "consumers"})}, // §535, §647, A-77, A-104: an ask no holder takes is bound by a key with its own time to live. {"bounds is key-ttl", keyTTL["bounds"]}, + // §725, A-469w, A-470: test-runs holds its runs' bound keys, each with its own time to live; 64 MiB its ceiling. + {"test-runs is key-ttl of 64 MiB", keyTTL["test-runs"] && stateMiB["test-runs"] == 64}, + {"the test runs' settings", runSettingsOf(raw)}, // RN25 (§469, §85): an old state section declares its cap, pipelines 256 MiB and kept 128 MiB. {"pipelines 256 MiB", stateMiB[seat.StatePipelines] == 256}, {"kept 128 MiB", stateMiB[seat.StateKept] == 128}, @@ -78,10 +81,13 @@ func TestManifest(t *testing.T) { {"consumes", equalSets(m.Consumes, []string{ "mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended", "operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended", - "node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", "mesh-wants.step-ended", "mesh-catalog.registration-changed", + "node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", + "node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged", // A-186: the seat list its write verbs' callers are checked against, heard as it changes. - "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"})}, + "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", + // §647 as A-466z amends it: no holder is read from the seat-holders view. + "seat:mesh-module-assigner.views.seat-holders"})}, // RN2 (§21, §195, §200, §77): the registrations and the catalog half asked by their verbs, the first views read and // asked again by hand (a hand act). {"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations", @@ -107,7 +113,8 @@ func TestManifest(t *testing.T) { "seat:node-build-node.accept.delivery", "seat:node-build-image.accept.check", "seat:node-build-image.accept.delivery"})}, {"uses operator-channel", equalSets(m.Uses, []string{"operator-channel"})}, // RN25 (§539): the runner's tools carry the prefix runner_. - {"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append"})}, + {"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append", + "runner_test_runs", "runner_test_runs_drop"})}, // A-305 (ADR 0227 rule 8, ADR 0240 rule 2): the process is judged by its unit, as every process of mesh-delivery, // never by a tool of its own; runner_health stays a tool. {"health is the unit", healthTool(raw) == "unit"}, @@ -175,3 +182,43 @@ func equalSets(a, b []string) bool { slices.Sort(y) return slices.Equal(x, y) } + +// runSettingsOf says whether the manifest holds the test runs' four settings (design 47 G §725; A-469w, A-469x): the cap a +// whole number of at most 64, the three times Go durations, each written into the file its environment names. +func runSettingsOf(raw []byte) bool { + var m struct { + Config map[string]map[string]any `json:"config"` + Resources []struct { + Type string `json:"type"` + Path string `json:"path"` + Content string `json:"content"` + Env map[string]string `json:"env"` + } `json:"resources"` + } + if json.Unmarshal(raw, &m) != nil { + return false + } + want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string"} + if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) { + return false + } + env := map[string]bool{} + for _, r := range m.Resources { + for _, v := range r.Env { + env[v] = true + } + } + for key, kind := range want { + if m.Config[key]["type"] != kind { + return false + } + found := false + for _, r := range m.Resources { + found = found || (r.Type == "file" && r.Content == "${config:"+key+"}" && env[r.Path]) + } + if !found { + return false + } + } + return true +} diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index 9198f2fa..893dcac0 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -18,7 +18,9 @@ "rerun", "accept", "cancel", - "ask-again" + "ask-again", + "test-runs", + "test-runs-drop" ] } ], @@ -27,7 +29,9 @@ "runner_first_view_again", "runner_health", "runner_unlogged", - "runner_unlogged_append" + "runner_unlogged_append", + "runner_test_runs", + "runner_test_runs_drop" ], "consumes": [ "mesh-delivery-indexer.take-in.taken", @@ -41,13 +45,17 @@ "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", + "node-build-go.step-progress", + "node-build-node.step-progress", + "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged", "mesh-catalog.seat-definition-changed", - "mesh-catalog.seat-definition-removed" + "mesh-catalog.seat-definition-removed", + "seat:mesh-module-assigner.views.seat-holders" ], "invokes": [ "seat:mesh-catalog.seat-definitions", @@ -79,6 +87,34 @@ "uses": [ "operator-channel" ], + "config": { + "test-runs-cap-mib": { + "type": "number", + "description": "MiB of the state test-runs at which the runner asks no new test run and ends the step could not run, naming the fill (design 47 G section 725, A-469d); the manifest's max-mib is the ceiling.", + "minimum": 1, + "maximum": 64, + "trusted": false, + "default": 16 + }, + "test-runs-kept": { + "type": "string", + "description": "how long an ended test run and its output object are kept after its end, a Go duration such as 168h (design 47 G section 725, A-469x, A-470a); at most 168h, so the pipeline naming its key is still kept.", + "trusted": false, + "default": "168h" + }, + "test-step-bound": { + "type": "string", + "description": "how long a taken test run's ask may say no progress before the run ends could not run, a Go duration such as 30m, counted from the build seat's ask-taken and again from each step-progress (design 47 G section 725, A-466n, A-469x).", + "trusted": false, + "default": "30m" + }, + "run-untaken-bound": { + "type": "string", + "description": "how long a test run's ask may wait untaken after the run's start before the run ends could not run, a Go duration such as 24h (design 47 G section 725, A-466s, A-469x).", + "trusted": false, + "default": "24h" + } + }, "state": [ { "name": "pipelines", @@ -107,6 +143,12 @@ "history": 1, "key-ttl": true }, + { + "name": "test-runs", + "history": 1, + "max-mib": 64, + "key-ttl": true + }, { "name": "findings", "history": 1 @@ -162,6 +204,38 @@ "place": ".", "unknown": "program" }, + { + "id": "test-runs-cap", + "type": "file", + "path": "${dir:state}/test-runs-cap-mib", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:test-runs-cap-mib}" + }, + { + "id": "test-runs-kept", + "type": "file", + "path": "${dir:state}/test-runs-kept", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:test-runs-kept}" + }, + { + "id": "test-step-bound", + "type": "file", + "path": "${dir:state}/test-step-bound", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:test-step-bound}" + }, + { + "id": "run-untaken-bound", + "type": "file", + "path": "${dir:state}/run-untaken-bound", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:run-untaken-bound}" + }, { "id": "pipeline", "type": "process", @@ -171,7 +245,11 @@ "./mesh-delivery-pipeline" ], "env": { - "MESH_BROKER_FILE": "${dir:state}/broker" + "MESH_BROKER_FILE": "${dir:state}/broker", + "MESH_RUNNER_TEST_RUNS_CAP_FILE": "${dir:state}/test-runs-cap-mib", + "MESH_RUNNER_TEST_RUNS_KEPT_FILE": "${dir:state}/test-runs-kept", + "MESH_RUNNER_TEST_STEP_BOUND_FILE": "${dir:state}/test-step-bound", + "MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound" }, "user": "mesh-delivery-pipeline", "health": {