From cd08f1e1c2f94d1d5fd04402a0c1c8a0b54625c0 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 11:31:26 +0100 Subject: [PATCH] runner: a test run is kept in the state test-runs with one clock, a cap and an age (R2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A test run is now one key of test-runs: its untaken bound written at its start, its clock at ask-taken and each step-progress, an expiry ending the run and every reader could not run and dropping its ask, a run past its bound while the runner was away ended at its start, the cap judged by the process, and an ended run aged with its output object in the step log's order. Builds: design 47 G §725, §720 (When the clock ends, A run never taken); A-466n, A-466p, A-466s, A-468b, A-468c, A-469d, A-469e, A-469w, A-469x, A-470, A-470a --- .../cmd/mesh-delivery-pipeline/main.go | 24 +- .../run_integration_test.go | 7 +- .../cmd/mesh-delivery-pipeline/testruns.go | 329 ++++++++++++++++++ .../mesh-delivery-pipeline/testruns_test.go | 58 +++ .../internal/pipeline/aged.go | 61 +++- .../internal/pipeline/place_test.go | 2 + .../internal/pipeline/runends.go | 326 +++++++++++++++++ .../internal/pipeline/start.go | 10 +- .../internal/pipeline/steps.go | 24 ++ .../internal/pipeline/testruns.go | 71 +++- .../internal/pipeline/testruns_test.go | 221 ++++++++++++ .../mesh-delivery-pipeline/manifest_test.go | 48 ++- modules/mesh-delivery-pipeline/module.json | 75 +++- 13 files changed, 1244 insertions(+), 12 deletions(-) create mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go create mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_test.go create mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/runends.go create mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 7d973c3d..17c35f90 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -234,13 +234,31 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro logger.Printf("the pipeline %s superseded %d older pipelines", newer.Pipeline, n) } } + // The test runs (§724, §725): one run per candidate, level and digest, kept in the state test-runs with its clock. + runDrop := pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings)) + runsKV, _, err := openTestRuns(ctx, js, cred, getenv, starter, hearer, runDrop, logger.Printf) + if err != nil { + return err + } + if err := watchRunBounds(ctx, runsKV, hearer.RunExpired, logger.Printf); err != nil { + return err + } + if err := hearStepProgress(heard, hearer.RunProgress, logger.Printf); err != nil { + return err + } if err := hearStepEnded(heard, hearer, logger.Printf); err != nil { return err } if err := hearToolchain(ctx, heard, toolchains, logger.Printf); err != nil { return err } - if err := hearAskTaken(ctx, heard, bounds, hearer.Taken, logger.Printf); err != nil { + taken := func(ask, node string) error { + if err := hearer.Taken(ask, node); err != nil { + return err + } + return hearer.RunTaken(ask, node) + } + if err := hearAskTaken(ctx, heard, bounds, taken, logger.Printf); err != nil { return err } expired := boundExpired(hearer, dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf) @@ -261,6 +279,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } // An entry whose step ended and was not appended is appended at the start, in order (§606, §549). hearer.AppendUnlogged() + // A test run whose bound passed while the runner was away ends now, as its expiry would have ended it (§725). + if n, err := hearer.TakeUp(); err != nil || n > 0 { + logger.Printf("the test runs taken up at the start: %d ended past their bound (%v)", n, err) + } go whileTheLogIsFull(ctx, fills, hearer, time.Minute) // The seat list, read over the calls connection and heard on the hub (seatlist.Part), which the seat's write verbs check // their callers against (A-186): registered before the hub starts, read again once its events are live. diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go index 25489f6a..32ee6038 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go @@ -65,8 +65,11 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes t.Fatal(err) } } - if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_bounds", History: 1, LimitMarkerTTL: time.Minute}); err != nil { - t.Fatal(err) + // bounds and test-runs keep keys with their own time to live (§725), as the bus makes them from the manifest's key-ttl. + for _, state := range []string{"bounds", TestRunsState} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil { + t.Fatal(err) + } } // The step log, the runner's old logs section (§470): the stream the bus makes for it. if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go new file mode 100644 index 00000000..ca1091ea --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go @@ -0,0 +1,329 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage" + nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// TestRunsState is the runner's state of its test runs (design 47 G §725), declared in its manifest's old state section. +const TestRunsState = "test-runs" + +// testRunsKV is the state test-runs on the bus: its runs by compare-and-set, and its bound keys, each written with its own +// time to live (A-77, A-470). +type testRunsKV struct { + ctx context.Context + kv jetstream.KeyValue +} + +func (t testRunsKV) Create(key string, r seat.TestRun) error { + raw, err := json.Marshal(r) + if err != nil { + return err + } + if _, err := t.kv.Create(t.ctx, key, raw); errors.Is(err, jetstream.ErrKeyExists) { + return pipeline.ErrExists + } else if err != nil { + return err + } + return nil +} + +func (t testRunsKV) Get(key string) (seat.TestRun, uint64, error) { + e, err := t.kv.Get(t.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return seat.TestRun{}, 0, pipeline.ErrNoRun + } + if err != nil { + return seat.TestRun{}, 0, err + } + var r seat.TestRun + if err := json.Unmarshal(e.Value(), &r); err != nil { + return seat.TestRun{}, 0, fmt.Errorf("the test run %s is no test run: %w", key, err) + } + return r, e.Revision(), nil +} + +func (t testRunsKV) Update(key string, r seat.TestRun, revision uint64) (uint64, error) { + raw, err := json.Marshal(r) + if err != nil { + return 0, err + } + rev, err := t.kv.Update(t.ctx, key, raw, revision) + if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) { + return 0, pipeline.ErrChanged + } + return rev, err +} + +func (t testRunsKV) Delete(key string) error { + if err := t.kv.Delete(t.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err + } + return nil +} + +// Bound writes key again with its time to live: deleted, then created with ttl, so its expiry is heard only ttl after +// the last write (a delete is no expiry). +func (t testRunsKV) Bound(key string, ttl time.Duration) error { + if err := t.Delete(key); err != nil { + return err + } + _, err := t.kv.Create(t.ctx, key, []byte(`{"bound":"`+key+`"}`), jetstream.KeyTTL(ttl)) + if errors.Is(err, jetstream.ErrKeyExists) { + return nil + } + return err +} + +func (t testRunsKV) Unbound(key string) error { return t.Delete(key) } + +func (t testRunsKV) Has(key string) (bool, error) { + _, err := t.kv.Get(t.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return false, nil + } + return err == nil, err +} + +func (t testRunsKV) Keys() ([]string, error) { + all, err := t.kv.Keys(t.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return all, err +} + +// runObjects is the state test-runs' own object store, `test-runs-objects`, made with the state (A-469e), opened when +// first used. +type runObjects struct { + ctx context.Context + js jetstream.JetStream + module string + mu sync.Mutex + store jetstream.ObjectStore +} + +func (o *runObjects) open() (jetstream.ObjectStore, error) { + o.mu.Lock() + defer o.mu.Unlock() + if o.store != nil { + return o.store, nil + } + s, err := o.js.ObjectStore(o.ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(TestRunsState))) + if err != nil { + return nil, fmt.Errorf("the object store of the state %s is not open: %w", TestRunsState, err) + } + o.store = s + return s, nil +} + +func (o *runObjects) Put(name string, data []byte) error { + s, err := o.open() + if err != nil { + return err + } + _, err = s.PutBytes(o.ctx, name, data) + return err +} + +func (o *runObjects) Get(name string) ([]byte, error) { + s, err := o.open() + if err != nil { + return nil, err + } + data, err := s.GetBytes(o.ctx, name) + if errors.Is(err, jetstream.ErrObjectNotFound) { + return nil, pipeline.ErrNoRun + } + return data, err +} + +func (o *runObjects) Delete(name string) error { + s, err := o.open() + if err != nil { + return err + } + if err := s.Delete(o.ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) { + return err + } + return nil +} + +// runsFill reads test-runs' fill, its key-value stream's bytes (A-469d): judged by the process against its cap setting. +func runsFill(ctx context.Context, js jetstream.JetStream, module string, now func() time.Time) func() (seat.Fill, error) { + return func() (seat.Fill, error) { + st, err := js.Stream(ctx, "KV_"+ownBucket(module, TestRunsState)) + if err != nil { + return seat.Fill{}, err + } + info, err := st.Info(ctx) + if err != nil { + return seat.Fill{}, err + } + at := now().UTC() + return seat.Fill{Store: TestRunsState, Bytes: int64(info.State.Bytes), Cap: info.Config.MaxBytes, At: &at}, nil + } +} + +// The files the manifest writes each setting of the test runs into, `${config:}`, named by the environment (A-469x). +const ( + envRunsCap = "MESH_RUNNER_TEST_RUNS_CAP_FILE" + envRunsKept = "MESH_RUNNER_TEST_RUNS_KEPT_FILE" + envStepBound = "MESH_RUNNER_TEST_STEP_BOUND_FILE" + envUntakenBound = "MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE" +) + +// maxRunsKept is test-runs-kept's maximum (A-470a): the pipeline naming a run's key is kept at least that long. +const maxRunsKept = 168 * time.Hour + +// runSettings reads the settings of the test runs at the start (§725; A-469x): test-runs-cap-mib a whole number of MiB, +// test-runs-kept, test-step-bound and run-untaken-bound Go durations. A setting whose file is missing or empty is its +// default; a content that is neither is an error naming the file. +func runSettings(getenv func(string) string) (pipeline.RunSettings, error) { + s := pipeline.DefaultRunSettings + if v, err := settingText(getenv(envRunsCap)); err != nil { + return s, err + } else if v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 1 { + return s, fmt.Errorf("%s holds %q, which is not a positive whole number", getenv(envRunsCap), v) + } + s.CapBytes = int64(n) << 20 + } + for _, d := range []struct { + env string + to *time.Duration + }{{envRunsKept, &s.Kept}, {envStepBound, &s.StepBound}, {envUntakenBound, &s.Untaken}} { + v, err := settingText(getenv(d.env)) + if err != nil { + return s, err + } + if v == "" { + continue + } + t, err := time.ParseDuration(v) + if err != nil || t <= 0 { + return s, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(d.env), v) + } + *d.to = t + } + if s.Kept > maxRunsKept { + return s, fmt.Errorf("%s holds %s, past test-runs-kept's maximum %s (A-470a)", getenv(envRunsKept), s.Kept, maxRunsKept) + } + return s, nil +} + +// settingText is one setting file's content, trimmed: empty for no path, no file or an empty one. +func settingText(path string) (string, error) { + if path == "" { + return "", nil + } + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return "", nil + } + if err != nil { + return "", err + } + return strings.TrimSpace(string(raw)), nil +} + +// progressSubjects hear every build seat's step-progress (§725), from their contracts. +var progressSubjects = []string{nodebuildgo.SubjectStepProgressAll, nodebuildnode.SubjectStepProgressAll, nodebuildimage.SubjectStepProgressAll} + +// hearStepProgress begins again the clock of the test run whose ask a build seat's holder says progress of (§725, A-466p). +func hearStepProgress(hub handlers, progress func(ask string) error, logf func(string, ...any)) error { + for _, subject := range progressSubjects { + err := hub.Handle(subject, func(m jetstream.Msg) { + var ev nodebuildgo.StepProgress + if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" { + logf("%s is no step-progress: %v", m.Subject(), err) + _ = m.Term() + return + } + if err := progress(ev.Ask); err != nil { + logf("the clock of the test run of ask %s is not begun again, heard again: %v", ev.Ask, err) + _ = m.Nak() + return + } + _ = m.Ack() + }) + if err != nil { + return err + } + } + return nil +} + +// watchRunBounds hears the expiry of each test run's bound key, the bus's marker of the key's end (§725): no timer in the +// process. +func watchRunBounds(ctx context.Context, kv jetstream.KeyValue, expired func(key string) error, logf func(string, ...any)) error { + w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) + if err != nil { + return err + } + go func() { + for { + select { + case <-ctx.Done(): + _ = w.Stop() + return + case e, ok := <-w.Updates(): + if !ok { + return + } + if e == nil || e.Operation() != jetstream.KeyValuePurge || !pipeline.IsRunBound(e.Key()) { + continue + } + if err := expired(e.Key()); err != nil { + logf("the test run of the bound %s that ended is not ended: %v", e.Key(), err) + } + } + } + }() + return nil +} + +// openTestRuns opens the state test-runs and wires it into the starter, the hearer and the ager (§725). +func openTestRuns(ctx context.Context, js jetstream.JetStream, cred process.Account, getenv func(string) string, + starter *pipeline.Starter, hearer *pipeline.Hearer, drop pipeline.Drop, logf func(string, ...any)) (jetstream.KeyValue, *runObjects, error) { + settings, err := runSettings(getenv) + if err != nil { + return nil, nil, err + } + kv, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, TestRunsState), "share a test run") + if err != nil { + return nil, nil, err + } + runs := testRunsKV{ctx: ctx, kv: kv} + objects := &runObjects{ctx: ctx, js: js, module: cred.Module} + starter.TestRuns, starter.Clock, starter.Runs = runs, runs, settings + starter.RunsFill = runsFill(ctx, js, cred.Module, time.Now) + starter.EndReaders = hearer.EndReaders + hearer.TestRuns, hearer.Clock, hearer.Runs, hearer.DropRun = runs, runs, settings, drop + hearer.RunOutputs = objects + if hearer.Ager != nil { + hearer.Ager.Runs, hearer.Ager.RunOutputs, hearer.Ager.RunsKept = runs, objects.Delete, settings.Kept + } + logf("test runs: cap %d bytes, kept %s, step bound %s, untaken bound %s", settings.CapBytes, settings.Kept, settings.StepBound, settings.Untaken) + return kv, objects, nil +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_test.go new file mode 100644 index 00000000..c8f14d81 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +// The test runs' settings reach the process through the files the manifest writes (design 47 G §725; A-469x, A-470a): the +// cap a whole number of MiB, the times Go durations, a missing file its default, a content that is neither, or a kept time +// past 168h, an error naming the file. +func TestTheTestRunsSettingsAreReadFromTheirFiles(t *testing.T) { + dir := t.TempDir() + write := func(name, content string) string { + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return p + } + rows := []struct { + name string + files map[string]string + ok bool + check func(t *testing.T, cap int64, kept, step, untaken time.Duration) + }{ + {"defaults", nil, true, func(t *testing.T, cap int64, kept, step, untaken time.Duration) { + if cap != 16<<20 || kept != 168*time.Hour || step != 30*time.Minute || untaken != 24*time.Hour { + t.Errorf("defaults = %d %s %s %s", cap, kept, step, untaken) + } + }}, + {"set", map[string]string{envRunsCap: "8\n", envRunsKept: "48h", envStepBound: "2s", envUntakenBound: "1m"}, true, + func(t *testing.T, cap int64, kept, step, untaken time.Duration) { + if cap != 8<<20 || kept != 48*time.Hour || step != 2*time.Second || untaken != time.Minute { + t.Errorf("settings = %d %s %s %s", cap, kept, step, untaken) + } + }}, + {"a cap that is no whole number", map[string]string{envRunsCap: "8.5"}, false, nil}, + {"a time that is no duration", map[string]string{envStepBound: "30"}, false, nil}, + {"kept past its maximum", map[string]string{envRunsKept: "169h"}, false, nil}, + } + for _, r := range rows { + t.Run(r.name, func(t *testing.T) { + env := map[string]string{} + for k, v := range r.files { + env[k] = write(r.name+k, v) + } + s, err := runSettings(func(k string) string { return env[k] }) + if (err == nil) != r.ok { + t.Fatalf("runSettings = %+v, %v", s, err) + } + if r.check != nil { + r.check(t, s.CapBytes, s.Kept, s.StepBound, s.Untaken) + } + }) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go index 487cc8c5..948ac76a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go @@ -53,7 +53,12 @@ type Ager struct { Get(key string) (seat.KeptResult, bool, error) Delete(key string) error } - Now func() time.Time + // Runs is the state test-runs, RunOutputs deletes an object of test-runs-objects and RunsKept is the setting + // test-runs-kept: an ended run is deleted with its output object once RunsKept has passed since its end (§725, A-470a). + Runs TestRuns + RunOutputs func(name string) error + RunsKept time.Duration + Now func() time.Time // Logf is the runner's journal: each delete writes a line naming the key or object and its retention. Logf func(string, ...any) } @@ -69,7 +74,16 @@ func (a *Ager) Age() { if a == nil || a.Log == nil { return } - for _, r := range []Retention{Retention7, Retention30, Retention90} { + retentions := []Retention{Retention7, Retention30, Retention90} + if a.Runs != nil { + // The test runs age first, in the same pass, so the pipeline that names a run's key is still kept (A-470a). + kept := a.RunsKept + if kept <= 0 { + kept = DefaultRunSettings.Kept + } + retentions = append([]Retention{{RetentionTestRuns, kept}}, retentions...) + } + for _, r := range retentions { if err := a.ageAt(r); err != nil { a.logf("aged: the data past %s are not aged: %v", r.Name, err) } @@ -103,8 +117,51 @@ func (a *Ager) ageAt(r Retention) error { return a.Aged.Put(r.Name, moved) } +// RetentionTestRuns is the name, in the state aged, of the test runs' retention, the setting test-runs-kept (§725). +const RetentionTestRuns = "test-runs" + +// ageRun deletes the test run the ended step of e read, and its output object, once its kept time has passed since the +// step's end, in the step log's order (§725, A-470a): the run's key is read from e's pipeline, its step's Shared run. +func (a *Ager) ageRun(r Retention, e seat.LogEntry) { + if e.What != seat.LogEntryWhatEnded { + return + } + p, _, err := a.Pipelines.Get(e.Pipeline) + if err != nil { + return + } + run := "" + for _, s := range p.Steps { + if s.Step == e.Step && s.Artifact == e.Artifact { + run = sharedRun(s) + } + } + if run == "" { + return + } + t, _, err := a.Runs.Get(run) + // The entry is past the retention (ageAt), and a run ends no later than the reader's end it was read at: a run not + // ended yet is aged at a later reader's end. + if err != nil || t.Ended == nil { + return + } + if t.Output != "" && a.RunOutputs != nil { + if err := a.RunOutputs(t.Output); err != nil { + a.logf("aged: the output %s of the test run %s is not deleted, so the run is kept: %v", t.Output, run, err) + return + } + } + if err := a.Runs.Delete(run); err != nil { + a.logf("aged: the test run %s is not deleted: %v", run, err) + return + } + a.logf("aged: deleted the test run %s and its output (%s after its end)", run, r.Age) +} + func (a *Ager) ageEntry(r Retention, e seat.LogEntry, now time.Time) { switch r.Name { + case RetentionTestRuns: + a.ageRun(r, e) case Retention7.Name: if e.Output != "" && a.Outputs != nil { if err := a.Outputs(e.Output); err != nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index b8240359..500075d4 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -355,6 +355,8 @@ func (m testRunsMem) Get(key string) (seat.TestRun, uint64, error) { return e.r, e.rev, nil } +func (m testRunsMem) Delete(key string) error { delete(m, key); return nil } + func (m testRunsMem) Update(key string, r seat.TestRun, rev uint64) (uint64, error) { if m[key].rev != rev { return 0, ErrChanged diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go new file mode 100644 index 00000000..7bfd4a94 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go @@ -0,0 +1,326 @@ +package pipeline + +import ( + "errors" + "fmt" + "strings" + "time" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// RunSettings are the runner's settings of its test runs (design 47 G §725; A-469d, A-469w, A-469x): the cap of test-runs +// the process judges a write by, how long an ended run is kept, the bound of a taken run's owed progress and the bound of +// an ask never taken. +type RunSettings struct { + CapBytes int64 + Kept time.Duration + StepBound time.Duration + Untaken time.Duration +} + +// The settings' defaults (§725; A-466n, A-466s): 16 MiB, 7 days, 30 minutes, 24 hours. +var DefaultRunSettings = RunSettings{CapBytes: 16 << 20, Kept: 7 * 24 * time.Hour, StepBound: 30 * time.Minute, Untaken: 24 * time.Hour} + +// The two bound keys of a test run in test-runs, each with its own time to live (§725; A-468b, A-470): `bound.`, +// written at the build seat's ask-taken of the run's ask and again at each step-progress of it, and `untaken.`, +// written at the run's start. +const ( + BoundPrefix = "bound." + UntakenPrefix = "untaken." +) + +// RunObjects is the state test-runs' own object store, test-runs-objects (A-469e): a run's output, kept while its key lives. +type RunObjects interface { + Put(name string, data []byte) error + // Get reads an object; ErrNoRun when it is gone. + Get(name string) ([]byte, error) + Delete(name string) error +} + +// RunClock is the bound keys of the test runs, keys of test-runs with their own time to live. +type RunClock interface { + // Bound writes key again with ttl as its time to live. + Bound(key string, ttl time.Duration) error + // Unbound deletes key; a key gone is no fault. + Unbound(key string) error + // Has says whether key is written. + Has(key string) (bool, error) + // Keys are test-runs' keys. + Keys() ([]string, error) +} + +// ParseRunBound reads a key of test-runs as a run's bound: the run's key and whether it is the untaken bound. +func ParseRunBound(key string) (run string, untaken, ok bool) { + if run, ok = strings.CutPrefix(key, BoundPrefix); ok { + return run, false, run != "" + } + if run, ok = strings.CutPrefix(key, UntakenPrefix); ok { + return run, true, run != "" + } + return "", false, false +} + +// IsRunBound says whether a key of test-runs is a bound key rather than a run. +func IsRunBound(key string) bool { + _, _, ok := ParseRunBound(key) + return ok +} + +// runOfAsk is the test run asked under ask: the run its asking step names in Shared, read from the state pipelines. +func (h *Hearer) runOfAsk(ask string) (string, bool) { + if h.TestRuns == nil || ask == "" { + return "", false + } + pkeys, err := h.Store.Keys("*.*.*") + if err != nil { + return "", false + } + for _, key := range pkeys { + p, _, err := h.Store.Get(key) + if err != nil { + continue + } + if i := stepOfAsk(p, ask); i >= 0 { + run := sharedRun(p.Steps[i]) + return run, run != "" + } + } + return "", false +} + +// RunTaken takes the ask-taken of a test run's ask (§725, A-470): the node that took it is kept on the run, its untaken +// bound deleted and its clock begun, `bound.` written with test-step-bound as its time to live. An ask of no test +// run takes nothing. +func (h *Hearer) RunTaken(ask, node string) error { + run, ok := h.runOfAsk(ask) + if !ok { + return nil + } + for { + r, rev, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) { + return nil + } + if err != nil { + return fmt.Errorf("the test run %s is not read: %w", run, err) + } + if r.Ask != ask || r.Ended != nil { + return nil + } + if r.Node != node { + r.Node = node + if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return fmt.Errorf("the test run %s is not written: %w", run, err) + } + } + break + } + if err := h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound); err != nil { + return fmt.Errorf("the clock of the test run %s is not begun: %w", run, err) + } + return h.Clock.Unbound(UntakenPrefix + run) +} + +// RunProgress takes a build seat's step-progress of a test run's ask (§725, A-466p): its clock begins again, so a run that +// keeps saying its progress is never ended by it. An ask of no running, taken test run takes nothing. +func (h *Hearer) RunProgress(ask string) error { + run, ok := h.runOfAsk(ask) + if !ok { + return nil + } + r, _, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) || (err == nil && (r.Ask != ask || r.Ended != nil)) { + return nil + } + if err != nil { + return fmt.Errorf("the test run %s is not read: %w", run, err) + } + return h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound) +} + +// RunExpired takes the expiry of a test run's bound key (§725, §720 *When the clock ends*, A-466s): the run ends `could not +// run`, naming its ask and the bound, its ask dropped and every reader ended with it. An untaken bound of a run whose ask +// was taken, and a bound of a run ended already, end nothing. +func (h *Hearer) RunExpired(key string) error { + run, untaken, ok := ParseRunBound(key) + if !ok || h.TestRuns == nil { + return nil + } + r, _, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) { + return nil + } + if err != nil { + return fmt.Errorf("the test run %s is not read: %w", run, err) + } + if r.Ended != nil || (untaken && r.Node != "") { + return nil + } + s := h.runSettings() + why := fmt.Sprintf("the ask %s of %s was taken by %s and said no progress within %s (test-step-bound)", r.Ask, r.Seat, r.Node, s.StepBound) + if untaken { + why = fmt.Sprintf("the ask %s of %s was not taken within %s of the run's start (run-untaken-bound)", r.Ask, r.Seat, s.Untaken) + } + if h.DropRun != nil { + if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil { + h.logf("the drop of ask %s of the test run %s is not answered: %v", r.Ask, run, err) + } + } + return h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil) +} + +// TakeUp takes up again, at the runner's start, each test run with no end (§725): a taken run whose clock is gone, and an +// untaken run whose untaken bound is gone, passed its bound while the runner was away and ends as its expiry would. It +// says how many it ended. +func (h *Hearer) TakeUp() (int, error) { + if h.TestRuns == nil || h.Clock == nil { + return 0, nil + } + all, err := h.Clock.Keys() + if err != nil { + return 0, fmt.Errorf("the test runs are not listed: %w", err) + } + ended := 0 + var first error + for _, run := range all { + if IsRunBound(run) { + continue + } + r, _, err := h.TestRuns.Get(run) + if err != nil || r.Ended != nil { + continue + } + bound := UntakenPrefix + run + if r.Node != "" { + bound = BoundPrefix + run + } + has, err := h.Clock.Has(bound) + if err != nil || has { + if err != nil && first == nil { + first = err + } + continue + } + if err := h.RunExpired(bound); err != nil && first == nil { + first = err + } + ended++ + } + return ended, first +} + +func (h *Hearer) runSettings() RunSettings { + s := h.Runs + d := DefaultRunSettings + if s.CapBytes <= 0 { + s.CapBytes = d.CapBytes + } + if s.Kept <= 0 { + s.Kept = d.Kept + } + if s.StepBound <= 0 { + s.StepBound = d.StepBound + } + if s.Untaken <= 0 { + s.Untaken = d.Untaken + } + return s +} + +// EndRun ends the test run under run with end's outcome, words, counts, node and output, written once by compare-and-set, +// and ends each of its readers with it but skip, the step whose own end ended it (§725, §726). A run ended already is +// not ended again. +func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) error { + var r seat.TestRun + for { + got, rev, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) { + return nil + } + if err != nil { + return fmt.Errorf("the test run %s is not read: %w", run, err) + } + if got.Ended != nil { + return nil + } + now := h.now() + got.Ended, got.Outcome, got.Said, got.Tests = &now, end.Outcome, end.Said, end.Tests + if end.Node != "" { + got.Node = end.Node + } + if end.Output != "" { + got.Output = end.Output + } + if _, err := h.TestRuns.Update(run, got, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return fmt.Errorf("the test run %s is not written: %w", run, err) + } + r = got + break + } + if h.Clock != nil { + for _, k := range []string{BoundPrefix + run, UntakenPrefix + run} { + if err := h.Clock.Unbound(k); err != nil { + h.logf("the bound %s of the ended test run is not deleted: %v", k, err) + } + } + } + var first error + for _, reader := range r.Readers { + if skip != nil && reader == *skip { + continue + } + if err := h.endReader(reader, run, r); err != nil && first == nil { + first = err + } + } + return first +} + +// endReader ends reader's step of run with the run's result (§726): its outcome, words, counts and node, the step naming +// the run in Shared as it did. A step ended already takes nothing. +func (h *Hearer) endReader(reader seat.TestRunReader, run string, r seat.TestRun) error { + for { + p, rev, err := h.Store.Get(reader.Pipeline) + if err != nil { + return nil // a pipeline gone has no reader left to end + } + i := -1 + for j, s := range p.Steps { + if s.Artifact == reader.Artifact && sharedRun(s) == run && s.Outcome == seat.StepResultOutcomeRunning { + i = j + } + } + if i < 0 { + return nil + } + now := h.now() + s := &p.Steps[i] + s.Outcome, s.Why, s.Node, s.Ended = r.Outcome, r.Said, r.Node, &now + if r.Tests != nil { + t := *r.Tests + s.Tests = &t + } + p.Unlogged = append(p.Unlogged, endEntry(reader.Pipeline, *s, now)) + asked, err := askApproval(reader.Pipeline, &p, now, h.Asker) + if err != nil { + return err + } + ended := endPipeline(&p, now) + if _, err := h.Store.Update(reader.Pipeline, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return fmt.Errorf("pipeline %s cannot be written: %w", reader.Pipeline, err) + } + h.Failures.StepEnded(h.Store, p, *s) + if err := h.say(seat.EventStepEnded, p, s); err != nil { + return fmt.Errorf("pipeline %s's %s is written but not said: %w", reader.Pipeline, seat.EventStepEnded, err) + } + h.appendUnlogged(reader.Pipeline) + return h.said(p, asked, ended) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index a2e793bd..e8ee14d2 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -84,8 +84,14 @@ type Starter struct { // Supersede ends the older running pipelines the pipeline just written supersedes (§31; Hearer.Supersede); nil ends none. Supersede func(newer seat.Pipeline) // TestRuns is the state test-runs, where a test step's run is shared by every step of its candidate, level and digest - // (§724, §725); nil shares none, each step's ask put. - TestRuns TestRuns + // (§724, §725); nil shares none, each step's ask put. Clock writes a run's untaken bound at its start, Runs holds the + // settings, RunsFill reads test-runs' fill, judged against Runs' cap before a run is written (A-469d), and EndReaders ends + // the steps of a run read ended or never asked. + TestRuns TestRuns + Clock RunClock + Runs RunSettings + RunsFill func() (seat.Fill, error) + EndReaders func(run string, readers []seat.TestRunReader, r seat.TestRun) error } // askSteps puts a placed pipeline's asks (§22, §24): a test run another step asked is read, not asked again (share), then diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index a1c7a3c2..654bea6b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -118,6 +118,26 @@ type Hearer struct { Fills *Fills // Widens computes the widens a pipeline-ended carries (§71, A-207); nil carries none. Widens func(seat.Pipeline) []seat.Widening + // TestRuns is the state test-runs and Clock its bound keys (§725); Runs the settings of its runs; DropRun drops a + // run's ask when its run ends unfinished (§119). A nil TestRuns keeps no test run. + TestRuns TestRuns + Clock RunClock + Runs RunSettings + DropRun Drop + // RunOutputs keeps a test run's output in test-runs-objects (§725, A-466v). + RunOutputs RunObjects +} + +// EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps +// of a run that was never asked. +func (h *Hearer) EndReaders(run string, readers []seat.TestRunReader, r seat.TestRun) error { + var first error + for _, reader := range readers { + if err := h.endReader(reader, run, r); err != nil && first == nil { + first = err + } + } + return first } func (h *Hearer) now() time.Time { @@ -154,6 +174,10 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s at %s (%s)", ErrRefused, ev.ID, ev.Step, ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint) case p.Steps[i].Outcome != seat.StepResultOutcomeRunning: + if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil { + // The run ended before its ask's end came (its bound, a drop): the end is ignored and said (§720, §725). + h.logf("the step-ended of ask %s is ignored: its test run %s ended before it (%s)", ev.ID, run, p.Steps[i].Why) + } return p, nil case !endOutcome(ev.Outcome): return p, fmt.Errorf("%w: ask %s ended %q, not an outcome of step-ended", ErrRefused, ev.ID, ev.Outcome) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go index 77b2e435..1b1b787b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go @@ -23,6 +23,8 @@ type TestRuns interface { Get(key string) (seat.TestRun, uint64, error) // Update writes key at the revision read; ErrChanged when another write came between. Update(key string, r seat.TestRun, revision uint64) (uint64, error) + // Delete deletes key; a key gone is no fault. + Delete(key string) error } // ErrNoRun is a Get of a test-runs key that is not written. @@ -127,7 +129,16 @@ func (s *Starter) share(key string, asks []Asked) []Asked { continue } run, ok, err := s.claimRun(a) + var full *fullErr switch { + case errors.As(err, &full): + // At the cap no run is asked: the steps end could not run, naming the fill (§725, A-469d). + s.logf("the test run %s of %s is not asked: %v", a.Run, key, err) + if s.EndReaders != nil { + if err := s.EndReaders(a.Run, a.Readers, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: err.Error()}); err != nil { + s.logf("the steps of the test run %s, not asked, are not ended: %v", a.Run, err) + } + } case err != nil: // A run that cannot be written or read is asked as the step's own, said: slower, never lost. s.logf("the test run %s of %s is not written, so the step is asked on its own: %v", a.Run, key, err) @@ -144,15 +155,67 @@ func (s *Starter) share(key string, asks []Asked) []Asked { return out } +// fullErr is a run not written because test-runs is at its cap (A-469d), naming the fill. +type fullErr struct{ fill seat.Fill } + +func (e *fullErr) Error() string { + return fmt.Sprintf("the state %s holds %d of its %d bytes, so no test run is asked (test-runs-cap-mib)", e.fill.Store, e.fill.Bytes, e.fill.Cap) +} + +// atCap is the error of test-runs at its cap setting, nil below it or when its fill is not read (said). +func (s *Starter) atCap() error { + if s.RunsFill == nil { + return nil + } + f, err := s.RunsFill() + if err != nil { + s.logf("the fill of test-runs is not read, so its cap is not judged: %v", err) + return nil + } + cap := s.Runs.CapBytes + if cap <= 0 { + cap = DefaultRunSettings.CapBytes + } + if f.Bytes >= cap { + f.Cap = cap + return &fullErr{fill: f} + } + return nil +} + +// started writes a new run and its untaken bound (§725, A-470). +func (s *Starter) started(a Asked, write func() error) error { + if err := s.atCap(); err != nil { + return err + } + if err := write(); err != nil { + return err + } + if s.Clock != nil { + untaken := s.Runs.Untaken + if untaken <= 0 { + untaken = DefaultRunSettings.Untaken + } + if err := s.Clock.Bound(UntakenPrefix+a.Run, untaken); err != nil { + s.logf("the untaken bound of the test run %s is not written: %v", a.Run, err) + } + } + return nil +} + // claimRun writes a's test run create-only; ok says the ask is put (the key is a's own), else run is the run it reads. func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) { now := s.now() value := seat.TestRun{Level: a.Level, Digest: a.Digest, Seat: a.Seat, Ask: a.ID, Started: &now, Readers: a.Readers} for { - err := s.TestRuns.Create(a.Run, value) + err := s.started(a, func() error { return s.TestRuns.Create(a.Run, value) }) if err == nil { return value, true, nil } + var full *fullErr + if errors.As(err, &full) { + return seat.TestRun{}, false, err + } if !errors.Is(err, ErrExists) { return seat.TestRun{}, false, err } @@ -167,7 +230,11 @@ func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) { return r, true, nil } if r.Ended != nil && r.Outcome == seat.StepResultOutcomeCouldNotRun { - if _, err := s.TestRuns.Update(a.Run, value, rev); errors.Is(err, ErrChanged) { + err := s.started(a, func() error { + _, err := s.TestRuns.Update(a.Run, value, rev) + return err + }) + if errors.Is(err, ErrChanged) { continue } else if err != nil { return seat.TestRun{}, false, err diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go new file mode 100644 index 00000000..aee903a9 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -0,0 +1,221 @@ +package pipeline + +import ( + "fmt" + "strings" + "testing" + "time" + + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// clockMem is the bound keys of test-runs in memory, each with the time to live it was last written with. +type clockMem struct { + runs testRunsMem + bound map[string]time.Duration +} + +func (c *clockMem) Bound(key string, ttl time.Duration) error { c.bound[key] = ttl; return nil } +func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil } +func (c *clockMem) Has(key string) (bool, error) { _, ok := c.bound[key]; return ok, nil } +func (c *clockMem) Keys() ([]string, error) { + var out []string + for k := range c.runs { + out = append(out, k) + } + for k := range c.bound { + out = append(out, k) + } + return out, nil +} + +type objectsMem map[string][]byte + +func (o objectsMem) Put(name string, data []byte) error { o[name] = data; return nil } +func (o objectsMem) Get(name string) ([]byte, error) { + if d, ok := o[name]; ok { + return d, nil + } + return nil, ErrNoRun +} +func (o objectsMem) Delete(name string) error { delete(o, name); return nil } + +// runFixture is a starter and a hearer over one pipelines store and one test-runs, the module's program and tools testing +// one directory. +type runFixture struct { + store memStore + runs testRunsMem + clock *clockMem + objects objectsMem + asks *stepAsks + drops []string + said []seat.StepEvent + logged []string + fill int64 + s *Starter + h *Hearer +} + +func newRunFixture() *runFixture { + f := &runFixture{store: memStore{}, runs: testRunsMem{}, objects: objectsMem{}, asks: &stepAsks{}} + f.clock = &clockMem{runs: f.runs, bound: map[string]time.Duration{}} + reg := catalog.Registration{Module: "app", Branches: []string{"main"}} + say := func(_ string, body any) error { + if e, ok := body.(seat.StepEvent); ok { + f.said = append(f.said, e) + } + return nil + } + f.h = &Hearer{Store: f.store, Say: say, TestRuns: f.runs, Clock: f.clock, RunOutputs: f.objects, + Logf: func(format string, a ...any) { f.logged = append(f.logged, fmt.Sprintf(format, a...)) }, + DropRun: func(seatName, node, ask, why string) error { + f.drops = append(f.drops, seatName+" "+node+" "+ask) + return nil + }} + f.s = &Starter{Pipelines: memCreated{f.store}, Store: f.store, TestRuns: f.runs, Clock: f.clock, Say: say, Steps: f.asks, + Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }, EndReaders: f.h.EndReaders, + RunsFill: func() (seat.Fill, error) { return seat.Fill{Store: "test-runs", Bytes: f.fill}, nil }, Logf: f.h.Logf} + return f +} + +func (f *runFixture) start(t *testing.T, source, revision int64) seat.Pipeline { + t.Helper() + entries := map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, + } + if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Module: "app", Commit: "c1", Fp: "src2:aa", Revision: revision, Entries: entries}); !ok || err != nil { + t.Fatalf("start = %v, %v", ok, err) + } + p, _, _ := f.store.Get(Key("main", source, "app")) + return p +} + +func (f *runFixture) pipeline(p seat.Pipeline) seat.Pipeline { + q, _, _ := f.store.Get(p.Pipeline) + return q +} + +// The test runs' clocks and cap (design 47 G §725; A-466n, A-466p, A-466s, A-468b, A-469d, A-470, A-470a): a run's untaken +// bound is written at its start, its clock at ask-taken and again at each step-progress; a clock's expiry ends the run and +// every reader could not run and drops its ask, a later step-ended ignored and journaled; an untaken bound of a taken run +// ends nothing; a run whose bound passed while the runner was away ends at its start; at the cap no run is asked; an ended +// run is aged with its output object in the step log's order. +func TestATestRunHasOneClockACapAndAnAge(t *testing.T) { + t.Run("clock", func(t *testing.T) { + f := newRunFixture() + f.h.Runs = RunSettings{StepBound: 30 * time.Minute, Untaken: 24 * time.Hour} + f.s.Runs = f.h.Runs + p := f.start(t, 4, 9) + unit := stepNamed(p, "app:program", "unit") + run := unit.Shared[0].Run + if f.clock.bound[UntakenPrefix+run] != 24*time.Hour || len(f.clock.bound) != 3 { + t.Fatalf("bounds at the start = %v, want the three runs' untaken bounds of 24h", f.clock.bound) + } + if err := f.h.RunTaken(unit.Ask, "ace"); err != nil { + t.Fatal(err) + } + if r, _, _ := f.runs.Get(run); r.Node != "ace" || f.clock.bound[BoundPrefix+run] != 30*time.Minute { + t.Fatalf("after ask-taken the run = %+v, bounds %v; want its node and its clock of 30m", r, f.clock.bound) + } + if _, ok := f.clock.bound[UntakenPrefix+run]; ok { + t.Error("the untaken bound of a taken run is kept") + } + // An untaken bound of a taken run ends nothing; progress begins the clock again. + if err := f.h.RunExpired(UntakenPrefix + run); err != nil { + t.Fatal(err) + } + delete(f.clock.bound, BoundPrefix+run) + if err := f.h.RunProgress(unit.Ask); err != nil || f.clock.bound[BoundPrefix+run] != 30*time.Minute { + t.Fatalf("progress = %v, bounds %v: want the clock begun again", err, f.clock.bound) + } + if r, _, _ := f.runs.Get(run); r.Ended != nil { + t.Fatal("a run is ended by its untaken bound after it was taken") + } + if err := f.h.RunExpired(BoundPrefix + run); err != nil { + t.Fatal(err) + } + r, _, _ := f.runs.Get(run) + if r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, unit.Ask) || !strings.Contains(r.Said, "30m") { + t.Errorf("the run whose clock ended = %+v, want could not run naming its ask and the bound", r) + } + if len(f.drops) != 1 || f.drops[0] != "node-build-go ace "+unit.Ask { + t.Errorf("drops = %v, want the run's ask dropped of ace", f.drops) + } + q := f.pipeline(p) + for _, art := range []string{"app:program", "app:tools"} { + if s := stepNamed(q, art, "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != r.Said { + t.Errorf("the %s unit step = %+v, want could not run with the run", art, s) + } + } + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask, + Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "passed", Ended: &ended}); err != nil { + t.Fatal(err) + } + if s := stepNamed(f.pipeline(p), "app:program", "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun { + t.Errorf("a step-ended after the run ended moved the step to %s", s.Outcome) + } + if !strings.Contains(strings.Join(f.logged, "\n"), "the step-ended of ask "+unit.Ask+" is ignored: its test run "+run) { + t.Errorf("the ignored step-ended is not journaled: %q", f.logged) + } + // An untaken run whose bound passed while the runner was away ends at the start. + lint := stepNamed(p, "app:program", "lint").Shared[0].Run + delete(f.clock.bound, UntakenPrefix+lint) + if n, err := f.h.TakeUp(); n != 1 || err != nil { + t.Fatalf("take-up ended %d, %v; want the lint run", n, err) + } + if r, _, _ := f.runs.Get(lint); r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "run-untaken-bound") { + t.Errorf("the lint run taken up = %+v", r) + } + }) + t.Run("cap", func(t *testing.T) { + f := newRunFixture() + f.s.Runs.CapBytes, f.fill = 1<<20, 1<<20 + p := f.start(t, 4, 9) + for _, a := range f.asks.ids { + if a == stepNamed(p, "app:program", "unit").Ask { + t.Error("a unit run is asked at the cap") + } + } + q := f.pipeline(p) + for _, art := range []string{"app:program", "app:tools"} { + if s := stepNamed(q, art, "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(s.Why, "1048576 of its 1048576 bytes") { + t.Errorf("the %s unit step at the cap = %+v, want could not run naming the fill", art, s) + } + } + }) + t.Run("age", func(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + unit := stepNamed(p, "app:program", "unit") + run := unit.Shared[0].Run + _ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed, Output: "out-1"}, nil) + f.objects["out-1"] = []byte("ok") + now := agedNow + a := &Ager{Runs: f.runs, RunOutputs: f.objects.Delete, RunsKept: 7 * 24 * time.Hour, Pipelines: agedDeleter{f.store}, + Aged: agedState{}, Now: func() time.Time { return now }, + Log: func(from uint64, limit int) ([]LoggedEntry, error) { + if from > 1 { + return nil, nil + } + return []LoggedEntry{{Sequence: 1, Entry: seat.LogEntry{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools", + What: seat.LogEntryWhatEnded, At: agedNow.Add(-6 * 24 * time.Hour)}}}, nil + }} + a.Age() + if _, _, err := f.runs.Get(run); err != nil { + t.Fatal("a run is aged before test-runs-kept passed") + } + now = now.Add(2 * 24 * time.Hour) + a.Age() + if _, _, err := f.runs.Get(run); err != ErrNoRun || len(f.objects) != 0 { + t.Errorf("the run past test-runs-kept is %v, objects %v; want both deleted", err, f.objects) + } + }) +} + +// agedDeleter is a memStore with the delete the ager makes. +type agedDeleter struct{ memStore } + +func (d agedDeleter) Delete(key string) error { delete(d.memStore, key); return nil } diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 80d88e8a..cc545315 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -55,9 +55,12 @@ func TestManifest(t *testing.T) { {"claims the seat serving every verb of its definition", len(m.Claims) == 1 && m.Claims[0].Name == seat.Seat && equalSets(m.Claims[0].Serves, seat.Verbs())}, // §27, §85, §330, §355, §77: the old state sections. - {"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "toolchains", "findings", "first-views", "consumers"})}, + {"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "test-runs", "toolchains", "findings", "first-views", "consumers"})}, // §535, §647, A-77, A-104: an ask no holder takes is bound by a key with its own time to live. {"bounds is key-ttl", keyTTL["bounds"]}, + // §725, A-469w, A-470: test-runs holds its runs' bound keys, each with its own time to live; 64 MiB its ceiling. + {"test-runs is key-ttl of 64 MiB", keyTTL["test-runs"] && stateMiB["test-runs"] == 64}, + {"the test runs' settings", runSettingsOf(raw)}, // RN25 (§469, §85): an old state section declares its cap, pipelines 256 MiB and kept 128 MiB. {"pipelines 256 MiB", stateMiB[seat.StatePipelines] == 256}, {"kept 128 MiB", stateMiB[seat.StateKept] == 128}, @@ -78,7 +81,8 @@ func TestManifest(t *testing.T) { {"consumes", equalSets(m.Consumes, []string{ "mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended", "operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended", - "node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", "mesh-wants.step-ended", "mesh-catalog.registration-changed", + "node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", + "node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged", // A-186: the seat list its write verbs' callers are checked against, heard as it changes. "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"})}, @@ -175,3 +179,43 @@ func equalSets(a, b []string) bool { slices.Sort(y) return slices.Equal(x, y) } + +// runSettingsOf says whether the manifest holds the test runs' four settings (design 47 G §725; A-469w, A-469x): the cap a +// whole number of at most 64, the three times Go durations, each written into the file its environment names. +func runSettingsOf(raw []byte) bool { + var m struct { + Config map[string]map[string]any `json:"config"` + Resources []struct { + Type string `json:"type"` + Path string `json:"path"` + Content string `json:"content"` + Env map[string]string `json:"env"` + } `json:"resources"` + } + if json.Unmarshal(raw, &m) != nil { + return false + } + want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string"} + if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) { + return false + } + env := map[string]bool{} + for _, r := range m.Resources { + for _, v := range r.Env { + env[v] = true + } + } + for key, kind := range want { + if m.Config[key]["type"] != kind { + return false + } + found := false + for _, r := range m.Resources { + found = found || (r.Type == "file" && r.Content == "${config:"+key+"}" && env[r.Path]) + } + if !found { + return false + } + } + return true +} diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index 9198f2fa..eec137ad 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -41,6 +41,9 @@ "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", + "node-build-go.step-progress", + "node-build-node.step-progress", + "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", @@ -79,6 +82,34 @@ "uses": [ "operator-channel" ], + "config": { + "test-runs-cap-mib": { + "type": "number", + "description": "MiB of the state test-runs at which the runner asks no new test run and ends the step could not run, naming the fill (design 47 G section 725, A-469d); the manifest's max-mib is the ceiling.", + "minimum": 1, + "maximum": 64, + "trusted": false, + "default": 16 + }, + "test-runs-kept": { + "type": "string", + "description": "how long an ended test run and its output object are kept after its end, a Go duration such as 168h (design 47 G section 725, A-469x, A-470a); at most 168h, so the pipeline naming its key is still kept.", + "trusted": false, + "default": "168h" + }, + "test-step-bound": { + "type": "string", + "description": "how long a taken test run's ask may say no progress before the run ends could not run, a Go duration such as 30m, counted from the build seat's ask-taken and again from each step-progress (design 47 G section 725, A-466n, A-469x).", + "trusted": false, + "default": "30m" + }, + "run-untaken-bound": { + "type": "string", + "description": "how long a test run's ask may wait untaken after the run's start before the run ends could not run, a Go duration such as 24h (design 47 G section 725, A-466s, A-469x).", + "trusted": false, + "default": "24h" + } + }, "state": [ { "name": "pipelines", @@ -107,6 +138,12 @@ "history": 1, "key-ttl": true }, + { + "name": "test-runs", + "history": 1, + "max-mib": 64, + "key-ttl": true + }, { "name": "findings", "history": 1 @@ -162,6 +199,38 @@ "place": ".", "unknown": "program" }, + { + "id": "test-runs-cap", + "type": "file", + "path": "${dir:state}/test-runs-cap-mib", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:test-runs-cap-mib}" + }, + { + "id": "test-runs-kept", + "type": "file", + "path": "${dir:state}/test-runs-kept", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:test-runs-kept}" + }, + { + "id": "test-step-bound", + "type": "file", + "path": "${dir:state}/test-step-bound", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:test-step-bound}" + }, + { + "id": "run-untaken-bound", + "type": "file", + "path": "${dir:state}/run-untaken-bound", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:run-untaken-bound}" + }, { "id": "pipeline", "type": "process", @@ -171,7 +240,11 @@ "./mesh-delivery-pipeline" ], "env": { - "MESH_BROKER_FILE": "${dir:state}/broker" + "MESH_BROKER_FILE": "${dir:state}/broker", + "MESH_RUNNER_TEST_RUNS_CAP_FILE": "${dir:state}/test-runs-cap-mib", + "MESH_RUNNER_TEST_RUNS_KEPT_FILE": "${dir:state}/test-runs-kept", + "MESH_RUNNER_TEST_STEP_BOUND_FILE": "${dir:state}/test-step-bound", + "MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound" }, "user": "mesh-delivery-pipeline", "health": {