diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 7d973c3d..17c35f90 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -234,13 +234,31 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro logger.Printf("the pipeline %s superseded %d older pipelines", newer.Pipeline, n) } } + // The test runs (§724, §725): one run per candidate, level and digest, kept in the state test-runs with its clock. + runDrop := pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings)) + runsKV, _, err := openTestRuns(ctx, js, cred, getenv, starter, hearer, runDrop, logger.Printf) + if err != nil { + return err + } + if err := watchRunBounds(ctx, runsKV, hearer.RunExpired, logger.Printf); err != nil { + return err + } + if err := hearStepProgress(heard, hearer.RunProgress, logger.Printf); err != nil { + return err + } if err := hearStepEnded(heard, hearer, logger.Printf); err != nil { return err } if err := hearToolchain(ctx, heard, toolchains, logger.Printf); err != nil { return err } - if err := hearAskTaken(ctx, heard, bounds, hearer.Taken, logger.Printf); err != nil { + taken := func(ask, node string) error { + if err := hearer.Taken(ask, node); err != nil { + return err + } + return hearer.RunTaken(ask, node) + } + if err := hearAskTaken(ctx, heard, bounds, taken, logger.Printf); err != nil { return err } expired := boundExpired(hearer, dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf) @@ -261,6 +279,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } // An entry whose step ended and was not appended is appended at the start, in order (§606, §549). hearer.AppendUnlogged() + // A test run whose bound passed while the runner was away ends now, as its expiry would have ended it (§725). + if n, err := hearer.TakeUp(); err != nil || n > 0 { + logger.Printf("the test runs taken up at the start: %d ended past their bound (%v)", n, err) + } go whileTheLogIsFull(ctx, fills, hearer, time.Minute) // The seat list, read over the calls connection and heard on the hub (seatlist.Part), which the seat's write verbs check // their callers against (A-186): registered before the hub starts, read again once its events are live. diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go index 25489f6a..32ee6038 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go @@ -65,8 +65,11 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes t.Fatal(err) } } - if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_bounds", History: 1, LimitMarkerTTL: time.Minute}); err != nil { - t.Fatal(err) + // bounds and test-runs keep keys with their own time to live (§725), as the bus makes them from the manifest's key-ttl. + for _, state := range []string{"bounds", TestRunsState} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil { + t.Fatal(err) + } } // The step log, the runner's old logs section (§470): the stream the bus makes for it. if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go new file mode 100644 index 00000000..ca1091ea --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go @@ -0,0 +1,329 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage" + nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// TestRunsState is the runner's state of its test runs (design 47 G §725), declared in its manifest's old state section. +const TestRunsState = "test-runs" + +// testRunsKV is the state test-runs on the bus: its runs by compare-and-set, and its bound keys, each written with its own +// time to live (A-77, A-470). +type testRunsKV struct { + ctx context.Context + kv jetstream.KeyValue +} + +func (t testRunsKV) Create(key string, r seat.TestRun) error { + raw, err := json.Marshal(r) + if err != nil { + return err + } + if _, err := t.kv.Create(t.ctx, key, raw); errors.Is(err, jetstream.ErrKeyExists) { + return pipeline.ErrExists + } else if err != nil { + return err + } + return nil +} + +func (t testRunsKV) Get(key string) (seat.TestRun, uint64, error) { + e, err := t.kv.Get(t.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return seat.TestRun{}, 0, pipeline.ErrNoRun + } + if err != nil { + return seat.TestRun{}, 0, err + } + var r seat.TestRun + if err := json.Unmarshal(e.Value(), &r); err != nil { + return seat.TestRun{}, 0, fmt.Errorf("the test run %s is no test run: %w", key, err) + } + return r, e.Revision(), nil +} + +func (t testRunsKV) Update(key string, r seat.TestRun, revision uint64) (uint64, error) { + raw, err := json.Marshal(r) + if err != nil { + return 0, err + } + rev, err := t.kv.Update(t.ctx, key, raw, revision) + if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) { + return 0, pipeline.ErrChanged + } + return rev, err +} + +func (t testRunsKV) Delete(key string) error { + if err := t.kv.Delete(t.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err + } + return nil +} + +// Bound writes key again with its time to live: deleted, then created with ttl, so its expiry is heard only ttl after +// the last write (a delete is no expiry). +func (t testRunsKV) Bound(key string, ttl time.Duration) error { + if err := t.Delete(key); err != nil { + return err + } + _, err := t.kv.Create(t.ctx, key, []byte(`{"bound":"`+key+`"}`), jetstream.KeyTTL(ttl)) + if errors.Is(err, jetstream.ErrKeyExists) { + return nil + } + return err +} + +func (t testRunsKV) Unbound(key string) error { return t.Delete(key) } + +func (t testRunsKV) Has(key string) (bool, error) { + _, err := t.kv.Get(t.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return false, nil + } + return err == nil, err +} + +func (t testRunsKV) Keys() ([]string, error) { + all, err := t.kv.Keys(t.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return all, err +} + +// runObjects is the state test-runs' own object store, `test-runs-objects`, made with the state (A-469e), opened when +// first used. +type runObjects struct { + ctx context.Context + js jetstream.JetStream + module string + mu sync.Mutex + store jetstream.ObjectStore +} + +func (o *runObjects) open() (jetstream.ObjectStore, error) { + o.mu.Lock() + defer o.mu.Unlock() + if o.store != nil { + return o.store, nil + } + s, err := o.js.ObjectStore(o.ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(TestRunsState))) + if err != nil { + return nil, fmt.Errorf("the object store of the state %s is not open: %w", TestRunsState, err) + } + o.store = s + return s, nil +} + +func (o *runObjects) Put(name string, data []byte) error { + s, err := o.open() + if err != nil { + return err + } + _, err = s.PutBytes(o.ctx, name, data) + return err +} + +func (o *runObjects) Get(name string) ([]byte, error) { + s, err := o.open() + if err != nil { + return nil, err + } + data, err := s.GetBytes(o.ctx, name) + if errors.Is(err, jetstream.ErrObjectNotFound) { + return nil, pipeline.ErrNoRun + } + return data, err +} + +func (o *runObjects) Delete(name string) error { + s, err := o.open() + if err != nil { + return err + } + if err := s.Delete(o.ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) { + return err + } + return nil +} + +// runsFill reads test-runs' fill, its key-value stream's bytes (A-469d): judged by the process against its cap setting. +func runsFill(ctx context.Context, js jetstream.JetStream, module string, now func() time.Time) func() (seat.Fill, error) { + return func() (seat.Fill, error) { + st, err := js.Stream(ctx, "KV_"+ownBucket(module, TestRunsState)) + if err != nil { + return seat.Fill{}, err + } + info, err := st.Info(ctx) + if err != nil { + return seat.Fill{}, err + } + at := now().UTC() + return seat.Fill{Store: TestRunsState, Bytes: int64(info.State.Bytes), Cap: info.Config.MaxBytes, At: &at}, nil + } +} + +// The files the manifest writes each setting of the test runs into, `${config:}`, named by the environment (A-469x). +const ( + envRunsCap = "MESH_RUNNER_TEST_RUNS_CAP_FILE" + envRunsKept = "MESH_RUNNER_TEST_RUNS_KEPT_FILE" + envStepBound = "MESH_RUNNER_TEST_STEP_BOUND_FILE" + envUntakenBound = "MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE" +) + +// maxRunsKept is test-runs-kept's maximum (A-470a): the pipeline naming a run's key is kept at least that long. +const maxRunsKept = 168 * time.Hour + +// runSettings reads the settings of the test runs at the start (§725; A-469x): test-runs-cap-mib a whole number of MiB, +// test-runs-kept, test-step-bound and run-untaken-bound Go durations. A setting whose file is missing or empty is its +// default; a content that is neither is an error naming the file. +func runSettings(getenv func(string) string) (pipeline.RunSettings, error) { + s := pipeline.DefaultRunSettings + if v, err := settingText(getenv(envRunsCap)); err != nil { + return s, err + } else if v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 1 { + return s, fmt.Errorf("%s holds %q, which is not a positive whole number", getenv(envRunsCap), v) + } + s.CapBytes = int64(n) << 20 + } + for _, d := range []struct { + env string + to *time.Duration + }{{envRunsKept, &s.Kept}, {envStepBound, &s.StepBound}, {envUntakenBound, &s.Untaken}} { + v, err := settingText(getenv(d.env)) + if err != nil { + return s, err + } + if v == "" { + continue + } + t, err := time.ParseDuration(v) + if err != nil || t <= 0 { + return s, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(d.env), v) + } + *d.to = t + } + if s.Kept > maxRunsKept { + return s, fmt.Errorf("%s holds %s, past test-runs-kept's maximum %s (A-470a)", getenv(envRunsKept), s.Kept, maxRunsKept) + } + return s, nil +} + +// settingText is one setting file's content, trimmed: empty for no path, no file or an empty one. +func settingText(path string) (string, error) { + if path == "" { + return "", nil + } + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return "", nil + } + if err != nil { + return "", err + } + return strings.TrimSpace(string(raw)), nil +} + +// progressSubjects hear every build seat's step-progress (§725), from their contracts. +var progressSubjects = []string{nodebuildgo.SubjectStepProgressAll, nodebuildnode.SubjectStepProgressAll, nodebuildimage.SubjectStepProgressAll} + +// hearStepProgress begins again the clock of the test run whose ask a build seat's holder says progress of (§725, A-466p). +func hearStepProgress(hub handlers, progress func(ask string) error, logf func(string, ...any)) error { + for _, subject := range progressSubjects { + err := hub.Handle(subject, func(m jetstream.Msg) { + var ev nodebuildgo.StepProgress + if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" { + logf("%s is no step-progress: %v", m.Subject(), err) + _ = m.Term() + return + } + if err := progress(ev.Ask); err != nil { + logf("the clock of the test run of ask %s is not begun again, heard again: %v", ev.Ask, err) + _ = m.Nak() + return + } + _ = m.Ack() + }) + if err != nil { + return err + } + } + return nil +} + +// watchRunBounds hears the expiry of each test run's bound key, the bus's marker of the key's end (§725): no timer in the +// process. +func watchRunBounds(ctx context.Context, kv jetstream.KeyValue, expired func(key string) error, logf func(string, ...any)) error { + w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) + if err != nil { + return err + } + go func() { + for { + select { + case <-ctx.Done(): + _ = w.Stop() + return + case e, ok := <-w.Updates(): + if !ok { + return + } + if e == nil || e.Operation() != jetstream.KeyValuePurge || !pipeline.IsRunBound(e.Key()) { + continue + } + if err := expired(e.Key()); err != nil { + logf("the test run of the bound %s that ended is not ended: %v", e.Key(), err) + } + } + } + }() + return nil +} + +// openTestRuns opens the state test-runs and wires it into the starter, the hearer and the ager (§725). +func openTestRuns(ctx context.Context, js jetstream.JetStream, cred process.Account, getenv func(string) string, + starter *pipeline.Starter, hearer *pipeline.Hearer, drop pipeline.Drop, logf func(string, ...any)) (jetstream.KeyValue, *runObjects, error) { + settings, err := runSettings(getenv) + if err != nil { + return nil, nil, err + } + kv, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, TestRunsState), "share a test run") + if err != nil { + return nil, nil, err + } + runs := testRunsKV{ctx: ctx, kv: kv} + objects := &runObjects{ctx: ctx, js: js, module: cred.Module} + starter.TestRuns, starter.Clock, starter.Runs = runs, runs, settings + starter.RunsFill = runsFill(ctx, js, cred.Module, time.Now) + starter.EndReaders = hearer.EndReaders + hearer.TestRuns, hearer.Clock, hearer.Runs, hearer.DropRun = runs, runs, settings, drop + hearer.RunOutputs = objects + if hearer.Ager != nil { + hearer.Ager.Runs, hearer.Ager.RunOutputs, hearer.Ager.RunsKept = runs, objects.Delete, settings.Kept + } + logf("test runs: cap %d bytes, kept %s, step bound %s, untaken bound %s", settings.CapBytes, settings.Kept, settings.StepBound, settings.Untaken) + return kv, objects, nil +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_test.go new file mode 100644 index 00000000..c8f14d81 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +// The test runs' settings reach the process through the files the manifest writes (design 47 G §725; A-469x, A-470a): the +// cap a whole number of MiB, the times Go durations, a missing file its default, a content that is neither, or a kept time +// past 168h, an error naming the file. +func TestTheTestRunsSettingsAreReadFromTheirFiles(t *testing.T) { + dir := t.TempDir() + write := func(name, content string) string { + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return p + } + rows := []struct { + name string + files map[string]string + ok bool + check func(t *testing.T, cap int64, kept, step, untaken time.Duration) + }{ + {"defaults", nil, true, func(t *testing.T, cap int64, kept, step, untaken time.Duration) { + if cap != 16<<20 || kept != 168*time.Hour || step != 30*time.Minute || untaken != 24*time.Hour { + t.Errorf("defaults = %d %s %s %s", cap, kept, step, untaken) + } + }}, + {"set", map[string]string{envRunsCap: "8\n", envRunsKept: "48h", envStepBound: "2s", envUntakenBound: "1m"}, true, + func(t *testing.T, cap int64, kept, step, untaken time.Duration) { + if cap != 8<<20 || kept != 48*time.Hour || step != 2*time.Second || untaken != time.Minute { + t.Errorf("settings = %d %s %s %s", cap, kept, step, untaken) + } + }}, + {"a cap that is no whole number", map[string]string{envRunsCap: "8.5"}, false, nil}, + {"a time that is no duration", map[string]string{envStepBound: "30"}, false, nil}, + {"kept past its maximum", map[string]string{envRunsKept: "169h"}, false, nil}, + } + for _, r := range rows { + t.Run(r.name, func(t *testing.T) { + env := map[string]string{} + for k, v := range r.files { + env[k] = write(r.name+k, v) + } + s, err := runSettings(func(k string) string { return env[k] }) + if (err == nil) != r.ok { + t.Fatalf("runSettings = %+v, %v", s, err) + } + if r.check != nil { + r.check(t, s.CapBytes, s.Kept, s.StepBound, s.Untaken) + } + }) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go index 487cc8c5..948ac76a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go @@ -53,7 +53,12 @@ type Ager struct { Get(key string) (seat.KeptResult, bool, error) Delete(key string) error } - Now func() time.Time + // Runs is the state test-runs, RunOutputs deletes an object of test-runs-objects and RunsKept is the setting + // test-runs-kept: an ended run is deleted with its output object once RunsKept has passed since its end (§725, A-470a). + Runs TestRuns + RunOutputs func(name string) error + RunsKept time.Duration + Now func() time.Time // Logf is the runner's journal: each delete writes a line naming the key or object and its retention. Logf func(string, ...any) } @@ -69,7 +74,16 @@ func (a *Ager) Age() { if a == nil || a.Log == nil { return } - for _, r := range []Retention{Retention7, Retention30, Retention90} { + retentions := []Retention{Retention7, Retention30, Retention90} + if a.Runs != nil { + // The test runs age first, in the same pass, so the pipeline that names a run's key is still kept (A-470a). + kept := a.RunsKept + if kept <= 0 { + kept = DefaultRunSettings.Kept + } + retentions = append([]Retention{{RetentionTestRuns, kept}}, retentions...) + } + for _, r := range retentions { if err := a.ageAt(r); err != nil { a.logf("aged: the data past %s are not aged: %v", r.Name, err) } @@ -103,8 +117,51 @@ func (a *Ager) ageAt(r Retention) error { return a.Aged.Put(r.Name, moved) } +// RetentionTestRuns is the name, in the state aged, of the test runs' retention, the setting test-runs-kept (§725). +const RetentionTestRuns = "test-runs" + +// ageRun deletes the test run the ended step of e read, and its output object, once its kept time has passed since the +// step's end, in the step log's order (§725, A-470a): the run's key is read from e's pipeline, its step's Shared run. +func (a *Ager) ageRun(r Retention, e seat.LogEntry) { + if e.What != seat.LogEntryWhatEnded { + return + } + p, _, err := a.Pipelines.Get(e.Pipeline) + if err != nil { + return + } + run := "" + for _, s := range p.Steps { + if s.Step == e.Step && s.Artifact == e.Artifact { + run = sharedRun(s) + } + } + if run == "" { + return + } + t, _, err := a.Runs.Get(run) + // The entry is past the retention (ageAt), and a run ends no later than the reader's end it was read at: a run not + // ended yet is aged at a later reader's end. + if err != nil || t.Ended == nil { + return + } + if t.Output != "" && a.RunOutputs != nil { + if err := a.RunOutputs(t.Output); err != nil { + a.logf("aged: the output %s of the test run %s is not deleted, so the run is kept: %v", t.Output, run, err) + return + } + } + if err := a.Runs.Delete(run); err != nil { + a.logf("aged: the test run %s is not deleted: %v", run, err) + return + } + a.logf("aged: deleted the test run %s and its output (%s after its end)", run, r.Age) +} + func (a *Ager) ageEntry(r Retention, e seat.LogEntry, now time.Time) { switch r.Name { + case RetentionTestRuns: + a.ageRun(r, e) case Retention7.Name: if e.Output != "" && a.Outputs != nil { if err := a.Outputs(e.Output); err != nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index b8240359..500075d4 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -355,6 +355,8 @@ func (m testRunsMem) Get(key string) (seat.TestRun, uint64, error) { return e.r, e.rev, nil } +func (m testRunsMem) Delete(key string) error { delete(m, key); return nil } + func (m testRunsMem) Update(key string, r seat.TestRun, rev uint64) (uint64, error) { if m[key].rev != rev { return 0, ErrChanged diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go new file mode 100644 index 00000000..7bfd4a94 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go @@ -0,0 +1,326 @@ +package pipeline + +import ( + "errors" + "fmt" + "strings" + "time" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// RunSettings are the runner's settings of its test runs (design 47 G §725; A-469d, A-469w, A-469x): the cap of test-runs +// the process judges a write by, how long an ended run is kept, the bound of a taken run's owed progress and the bound of +// an ask never taken. +type RunSettings struct { + CapBytes int64 + Kept time.Duration + StepBound time.Duration + Untaken time.Duration +} + +// The settings' defaults (§725; A-466n, A-466s): 16 MiB, 7 days, 30 minutes, 24 hours. +var DefaultRunSettings = RunSettings{CapBytes: 16 << 20, Kept: 7 * 24 * time.Hour, StepBound: 30 * time.Minute, Untaken: 24 * time.Hour} + +// The two bound keys of a test run in test-runs, each with its own time to live (§725; A-468b, A-470): `bound.`, +// written at the build seat's ask-taken of the run's ask and again at each step-progress of it, and `untaken.`, +// written at the run's start. +const ( + BoundPrefix = "bound." + UntakenPrefix = "untaken." +) + +// RunObjects is the state test-runs' own object store, test-runs-objects (A-469e): a run's output, kept while its key lives. +type RunObjects interface { + Put(name string, data []byte) error + // Get reads an object; ErrNoRun when it is gone. + Get(name string) ([]byte, error) + Delete(name string) error +} + +// RunClock is the bound keys of the test runs, keys of test-runs with their own time to live. +type RunClock interface { + // Bound writes key again with ttl as its time to live. + Bound(key string, ttl time.Duration) error + // Unbound deletes key; a key gone is no fault. + Unbound(key string) error + // Has says whether key is written. + Has(key string) (bool, error) + // Keys are test-runs' keys. + Keys() ([]string, error) +} + +// ParseRunBound reads a key of test-runs as a run's bound: the run's key and whether it is the untaken bound. +func ParseRunBound(key string) (run string, untaken, ok bool) { + if run, ok = strings.CutPrefix(key, BoundPrefix); ok { + return run, false, run != "" + } + if run, ok = strings.CutPrefix(key, UntakenPrefix); ok { + return run, true, run != "" + } + return "", false, false +} + +// IsRunBound says whether a key of test-runs is a bound key rather than a run. +func IsRunBound(key string) bool { + _, _, ok := ParseRunBound(key) + return ok +} + +// runOfAsk is the test run asked under ask: the run its asking step names in Shared, read from the state pipelines. +func (h *Hearer) runOfAsk(ask string) (string, bool) { + if h.TestRuns == nil || ask == "" { + return "", false + } + pkeys, err := h.Store.Keys("*.*.*") + if err != nil { + return "", false + } + for _, key := range pkeys { + p, _, err := h.Store.Get(key) + if err != nil { + continue + } + if i := stepOfAsk(p, ask); i >= 0 { + run := sharedRun(p.Steps[i]) + return run, run != "" + } + } + return "", false +} + +// RunTaken takes the ask-taken of a test run's ask (§725, A-470): the node that took it is kept on the run, its untaken +// bound deleted and its clock begun, `bound.` written with test-step-bound as its time to live. An ask of no test +// run takes nothing. +func (h *Hearer) RunTaken(ask, node string) error { + run, ok := h.runOfAsk(ask) + if !ok { + return nil + } + for { + r, rev, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) { + return nil + } + if err != nil { + return fmt.Errorf("the test run %s is not read: %w", run, err) + } + if r.Ask != ask || r.Ended != nil { + return nil + } + if r.Node != node { + r.Node = node + if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return fmt.Errorf("the test run %s is not written: %w", run, err) + } + } + break + } + if err := h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound); err != nil { + return fmt.Errorf("the clock of the test run %s is not begun: %w", run, err) + } + return h.Clock.Unbound(UntakenPrefix + run) +} + +// RunProgress takes a build seat's step-progress of a test run's ask (§725, A-466p): its clock begins again, so a run that +// keeps saying its progress is never ended by it. An ask of no running, taken test run takes nothing. +func (h *Hearer) RunProgress(ask string) error { + run, ok := h.runOfAsk(ask) + if !ok { + return nil + } + r, _, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) || (err == nil && (r.Ask != ask || r.Ended != nil)) { + return nil + } + if err != nil { + return fmt.Errorf("the test run %s is not read: %w", run, err) + } + return h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound) +} + +// RunExpired takes the expiry of a test run's bound key (§725, §720 *When the clock ends*, A-466s): the run ends `could not +// run`, naming its ask and the bound, its ask dropped and every reader ended with it. An untaken bound of a run whose ask +// was taken, and a bound of a run ended already, end nothing. +func (h *Hearer) RunExpired(key string) error { + run, untaken, ok := ParseRunBound(key) + if !ok || h.TestRuns == nil { + return nil + } + r, _, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) { + return nil + } + if err != nil { + return fmt.Errorf("the test run %s is not read: %w", run, err) + } + if r.Ended != nil || (untaken && r.Node != "") { + return nil + } + s := h.runSettings() + why := fmt.Sprintf("the ask %s of %s was taken by %s and said no progress within %s (test-step-bound)", r.Ask, r.Seat, r.Node, s.StepBound) + if untaken { + why = fmt.Sprintf("the ask %s of %s was not taken within %s of the run's start (run-untaken-bound)", r.Ask, r.Seat, s.Untaken) + } + if h.DropRun != nil { + if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil { + h.logf("the drop of ask %s of the test run %s is not answered: %v", r.Ask, run, err) + } + } + return h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil) +} + +// TakeUp takes up again, at the runner's start, each test run with no end (§725): a taken run whose clock is gone, and an +// untaken run whose untaken bound is gone, passed its bound while the runner was away and ends as its expiry would. It +// says how many it ended. +func (h *Hearer) TakeUp() (int, error) { + if h.TestRuns == nil || h.Clock == nil { + return 0, nil + } + all, err := h.Clock.Keys() + if err != nil { + return 0, fmt.Errorf("the test runs are not listed: %w", err) + } + ended := 0 + var first error + for _, run := range all { + if IsRunBound(run) { + continue + } + r, _, err := h.TestRuns.Get(run) + if err != nil || r.Ended != nil { + continue + } + bound := UntakenPrefix + run + if r.Node != "" { + bound = BoundPrefix + run + } + has, err := h.Clock.Has(bound) + if err != nil || has { + if err != nil && first == nil { + first = err + } + continue + } + if err := h.RunExpired(bound); err != nil && first == nil { + first = err + } + ended++ + } + return ended, first +} + +func (h *Hearer) runSettings() RunSettings { + s := h.Runs + d := DefaultRunSettings + if s.CapBytes <= 0 { + s.CapBytes = d.CapBytes + } + if s.Kept <= 0 { + s.Kept = d.Kept + } + if s.StepBound <= 0 { + s.StepBound = d.StepBound + } + if s.Untaken <= 0 { + s.Untaken = d.Untaken + } + return s +} + +// EndRun ends the test run under run with end's outcome, words, counts, node and output, written once by compare-and-set, +// and ends each of its readers with it but skip, the step whose own end ended it (§725, §726). A run ended already is +// not ended again. +func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) error { + var r seat.TestRun + for { + got, rev, err := h.TestRuns.Get(run) + if errors.Is(err, ErrNoRun) { + return nil + } + if err != nil { + return fmt.Errorf("the test run %s is not read: %w", run, err) + } + if got.Ended != nil { + return nil + } + now := h.now() + got.Ended, got.Outcome, got.Said, got.Tests = &now, end.Outcome, end.Said, end.Tests + if end.Node != "" { + got.Node = end.Node + } + if end.Output != "" { + got.Output = end.Output + } + if _, err := h.TestRuns.Update(run, got, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return fmt.Errorf("the test run %s is not written: %w", run, err) + } + r = got + break + } + if h.Clock != nil { + for _, k := range []string{BoundPrefix + run, UntakenPrefix + run} { + if err := h.Clock.Unbound(k); err != nil { + h.logf("the bound %s of the ended test run is not deleted: %v", k, err) + } + } + } + var first error + for _, reader := range r.Readers { + if skip != nil && reader == *skip { + continue + } + if err := h.endReader(reader, run, r); err != nil && first == nil { + first = err + } + } + return first +} + +// endReader ends reader's step of run with the run's result (§726): its outcome, words, counts and node, the step naming +// the run in Shared as it did. A step ended already takes nothing. +func (h *Hearer) endReader(reader seat.TestRunReader, run string, r seat.TestRun) error { + for { + p, rev, err := h.Store.Get(reader.Pipeline) + if err != nil { + return nil // a pipeline gone has no reader left to end + } + i := -1 + for j, s := range p.Steps { + if s.Artifact == reader.Artifact && sharedRun(s) == run && s.Outcome == seat.StepResultOutcomeRunning { + i = j + } + } + if i < 0 { + return nil + } + now := h.now() + s := &p.Steps[i] + s.Outcome, s.Why, s.Node, s.Ended = r.Outcome, r.Said, r.Node, &now + if r.Tests != nil { + t := *r.Tests + s.Tests = &t + } + p.Unlogged = append(p.Unlogged, endEntry(reader.Pipeline, *s, now)) + asked, err := askApproval(reader.Pipeline, &p, now, h.Asker) + if err != nil { + return err + } + ended := endPipeline(&p, now) + if _, err := h.Store.Update(reader.Pipeline, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return fmt.Errorf("pipeline %s cannot be written: %w", reader.Pipeline, err) + } + h.Failures.StepEnded(h.Store, p, *s) + if err := h.say(seat.EventStepEnded, p, s); err != nil { + return fmt.Errorf("pipeline %s's %s is written but not said: %w", reader.Pipeline, seat.EventStepEnded, err) + } + h.appendUnlogged(reader.Pipeline) + return h.said(p, asked, ended) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index a2e793bd..e8ee14d2 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -84,8 +84,14 @@ type Starter struct { // Supersede ends the older running pipelines the pipeline just written supersedes (§31; Hearer.Supersede); nil ends none. Supersede func(newer seat.Pipeline) // TestRuns is the state test-runs, where a test step's run is shared by every step of its candidate, level and digest - // (§724, §725); nil shares none, each step's ask put. - TestRuns TestRuns + // (§724, §725); nil shares none, each step's ask put. Clock writes a run's untaken bound at its start, Runs holds the + // settings, RunsFill reads test-runs' fill, judged against Runs' cap before a run is written (A-469d), and EndReaders ends + // the steps of a run read ended or never asked. + TestRuns TestRuns + Clock RunClock + Runs RunSettings + RunsFill func() (seat.Fill, error) + EndReaders func(run string, readers []seat.TestRunReader, r seat.TestRun) error } // askSteps puts a placed pipeline's asks (§22, §24): a test run another step asked is read, not asked again (share), then diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index a1c7a3c2..654bea6b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -118,6 +118,26 @@ type Hearer struct { Fills *Fills // Widens computes the widens a pipeline-ended carries (§71, A-207); nil carries none. Widens func(seat.Pipeline) []seat.Widening + // TestRuns is the state test-runs and Clock its bound keys (§725); Runs the settings of its runs; DropRun drops a + // run's ask when its run ends unfinished (§119). A nil TestRuns keeps no test run. + TestRuns TestRuns + Clock RunClock + Runs RunSettings + DropRun Drop + // RunOutputs keeps a test run's output in test-runs-objects (§725, A-466v). + RunOutputs RunObjects +} + +// EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps +// of a run that was never asked. +func (h *Hearer) EndReaders(run string, readers []seat.TestRunReader, r seat.TestRun) error { + var first error + for _, reader := range readers { + if err := h.endReader(reader, run, r); err != nil && first == nil { + first = err + } + } + return first } func (h *Hearer) now() time.Time { @@ -154,6 +174,10 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s at %s (%s)", ErrRefused, ev.ID, ev.Step, ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint) case p.Steps[i].Outcome != seat.StepResultOutcomeRunning: + if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil { + // The run ended before its ask's end came (its bound, a drop): the end is ignored and said (§720, §725). + h.logf("the step-ended of ask %s is ignored: its test run %s ended before it (%s)", ev.ID, run, p.Steps[i].Why) + } return p, nil case !endOutcome(ev.Outcome): return p, fmt.Errorf("%w: ask %s ended %q, not an outcome of step-ended", ErrRefused, ev.ID, ev.Outcome) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go index 77b2e435..1b1b787b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go @@ -23,6 +23,8 @@ type TestRuns interface { Get(key string) (seat.TestRun, uint64, error) // Update writes key at the revision read; ErrChanged when another write came between. Update(key string, r seat.TestRun, revision uint64) (uint64, error) + // Delete deletes key; a key gone is no fault. + Delete(key string) error } // ErrNoRun is a Get of a test-runs key that is not written. @@ -127,7 +129,16 @@ func (s *Starter) share(key string, asks []Asked) []Asked { continue } run, ok, err := s.claimRun(a) + var full *fullErr switch { + case errors.As(err, &full): + // At the cap no run is asked: the steps end could not run, naming the fill (§725, A-469d). + s.logf("the test run %s of %s is not asked: %v", a.Run, key, err) + if s.EndReaders != nil { + if err := s.EndReaders(a.Run, a.Readers, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: err.Error()}); err != nil { + s.logf("the steps of the test run %s, not asked, are not ended: %v", a.Run, err) + } + } case err != nil: // A run that cannot be written or read is asked as the step's own, said: slower, never lost. s.logf("the test run %s of %s is not written, so the step is asked on its own: %v", a.Run, key, err) @@ -144,15 +155,67 @@ func (s *Starter) share(key string, asks []Asked) []Asked { return out } +// fullErr is a run not written because test-runs is at its cap (A-469d), naming the fill. +type fullErr struct{ fill seat.Fill } + +func (e *fullErr) Error() string { + return fmt.Sprintf("the state %s holds %d of its %d bytes, so no test run is asked (test-runs-cap-mib)", e.fill.Store, e.fill.Bytes, e.fill.Cap) +} + +// atCap is the error of test-runs at its cap setting, nil below it or when its fill is not read (said). +func (s *Starter) atCap() error { + if s.RunsFill == nil { + return nil + } + f, err := s.RunsFill() + if err != nil { + s.logf("the fill of test-runs is not read, so its cap is not judged: %v", err) + return nil + } + cap := s.Runs.CapBytes + if cap <= 0 { + cap = DefaultRunSettings.CapBytes + } + if f.Bytes >= cap { + f.Cap = cap + return &fullErr{fill: f} + } + return nil +} + +// started writes a new run and its untaken bound (§725, A-470). +func (s *Starter) started(a Asked, write func() error) error { + if err := s.atCap(); err != nil { + return err + } + if err := write(); err != nil { + return err + } + if s.Clock != nil { + untaken := s.Runs.Untaken + if untaken <= 0 { + untaken = DefaultRunSettings.Untaken + } + if err := s.Clock.Bound(UntakenPrefix+a.Run, untaken); err != nil { + s.logf("the untaken bound of the test run %s is not written: %v", a.Run, err) + } + } + return nil +} + // claimRun writes a's test run create-only; ok says the ask is put (the key is a's own), else run is the run it reads. func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) { now := s.now() value := seat.TestRun{Level: a.Level, Digest: a.Digest, Seat: a.Seat, Ask: a.ID, Started: &now, Readers: a.Readers} for { - err := s.TestRuns.Create(a.Run, value) + err := s.started(a, func() error { return s.TestRuns.Create(a.Run, value) }) if err == nil { return value, true, nil } + var full *fullErr + if errors.As(err, &full) { + return seat.TestRun{}, false, err + } if !errors.Is(err, ErrExists) { return seat.TestRun{}, false, err } @@ -167,7 +230,11 @@ func (s *Starter) claimRun(a Asked) (seat.TestRun, bool, error) { return r, true, nil } if r.Ended != nil && r.Outcome == seat.StepResultOutcomeCouldNotRun { - if _, err := s.TestRuns.Update(a.Run, value, rev); errors.Is(err, ErrChanged) { + err := s.started(a, func() error { + _, err := s.TestRuns.Update(a.Run, value, rev) + return err + }) + if errors.Is(err, ErrChanged) { continue } else if err != nil { return seat.TestRun{}, false, err diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go new file mode 100644 index 00000000..aee903a9 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -0,0 +1,221 @@ +package pipeline + +import ( + "fmt" + "strings" + "testing" + "time" + + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// clockMem is the bound keys of test-runs in memory, each with the time to live it was last written with. +type clockMem struct { + runs testRunsMem + bound map[string]time.Duration +} + +func (c *clockMem) Bound(key string, ttl time.Duration) error { c.bound[key] = ttl; return nil } +func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil } +func (c *clockMem) Has(key string) (bool, error) { _, ok := c.bound[key]; return ok, nil } +func (c *clockMem) Keys() ([]string, error) { + var out []string + for k := range c.runs { + out = append(out, k) + } + for k := range c.bound { + out = append(out, k) + } + return out, nil +} + +type objectsMem map[string][]byte + +func (o objectsMem) Put(name string, data []byte) error { o[name] = data; return nil } +func (o objectsMem) Get(name string) ([]byte, error) { + if d, ok := o[name]; ok { + return d, nil + } + return nil, ErrNoRun +} +func (o objectsMem) Delete(name string) error { delete(o, name); return nil } + +// runFixture is a starter and a hearer over one pipelines store and one test-runs, the module's program and tools testing +// one directory. +type runFixture struct { + store memStore + runs testRunsMem + clock *clockMem + objects objectsMem + asks *stepAsks + drops []string + said []seat.StepEvent + logged []string + fill int64 + s *Starter + h *Hearer +} + +func newRunFixture() *runFixture { + f := &runFixture{store: memStore{}, runs: testRunsMem{}, objects: objectsMem{}, asks: &stepAsks{}} + f.clock = &clockMem{runs: f.runs, bound: map[string]time.Duration{}} + reg := catalog.Registration{Module: "app", Branches: []string{"main"}} + say := func(_ string, body any) error { + if e, ok := body.(seat.StepEvent); ok { + f.said = append(f.said, e) + } + return nil + } + f.h = &Hearer{Store: f.store, Say: say, TestRuns: f.runs, Clock: f.clock, RunOutputs: f.objects, + Logf: func(format string, a ...any) { f.logged = append(f.logged, fmt.Sprintf(format, a...)) }, + DropRun: func(seatName, node, ask, why string) error { + f.drops = append(f.drops, seatName+" "+node+" "+ask) + return nil + }} + f.s = &Starter{Pipelines: memCreated{f.store}, Store: f.store, TestRuns: f.runs, Clock: f.clock, Say: say, Steps: f.asks, + Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }, EndReaders: f.h.EndReaders, + RunsFill: func() (seat.Fill, error) { return seat.Fill{Store: "test-runs", Bytes: f.fill}, nil }, Logf: f.h.Logf} + return f +} + +func (f *runFixture) start(t *testing.T, source, revision int64) seat.Pipeline { + t.Helper() + entries := map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, + } + if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Module: "app", Commit: "c1", Fp: "src2:aa", Revision: revision, Entries: entries}); !ok || err != nil { + t.Fatalf("start = %v, %v", ok, err) + } + p, _, _ := f.store.Get(Key("main", source, "app")) + return p +} + +func (f *runFixture) pipeline(p seat.Pipeline) seat.Pipeline { + q, _, _ := f.store.Get(p.Pipeline) + return q +} + +// The test runs' clocks and cap (design 47 G §725; A-466n, A-466p, A-466s, A-468b, A-469d, A-470, A-470a): a run's untaken +// bound is written at its start, its clock at ask-taken and again at each step-progress; a clock's expiry ends the run and +// every reader could not run and drops its ask, a later step-ended ignored and journaled; an untaken bound of a taken run +// ends nothing; a run whose bound passed while the runner was away ends at its start; at the cap no run is asked; an ended +// run is aged with its output object in the step log's order. +func TestATestRunHasOneClockACapAndAnAge(t *testing.T) { + t.Run("clock", func(t *testing.T) { + f := newRunFixture() + f.h.Runs = RunSettings{StepBound: 30 * time.Minute, Untaken: 24 * time.Hour} + f.s.Runs = f.h.Runs + p := f.start(t, 4, 9) + unit := stepNamed(p, "app:program", "unit") + run := unit.Shared[0].Run + if f.clock.bound[UntakenPrefix+run] != 24*time.Hour || len(f.clock.bound) != 3 { + t.Fatalf("bounds at the start = %v, want the three runs' untaken bounds of 24h", f.clock.bound) + } + if err := f.h.RunTaken(unit.Ask, "ace"); err != nil { + t.Fatal(err) + } + if r, _, _ := f.runs.Get(run); r.Node != "ace" || f.clock.bound[BoundPrefix+run] != 30*time.Minute { + t.Fatalf("after ask-taken the run = %+v, bounds %v; want its node and its clock of 30m", r, f.clock.bound) + } + if _, ok := f.clock.bound[UntakenPrefix+run]; ok { + t.Error("the untaken bound of a taken run is kept") + } + // An untaken bound of a taken run ends nothing; progress begins the clock again. + if err := f.h.RunExpired(UntakenPrefix + run); err != nil { + t.Fatal(err) + } + delete(f.clock.bound, BoundPrefix+run) + if err := f.h.RunProgress(unit.Ask); err != nil || f.clock.bound[BoundPrefix+run] != 30*time.Minute { + t.Fatalf("progress = %v, bounds %v: want the clock begun again", err, f.clock.bound) + } + if r, _, _ := f.runs.Get(run); r.Ended != nil { + t.Fatal("a run is ended by its untaken bound after it was taken") + } + if err := f.h.RunExpired(BoundPrefix + run); err != nil { + t.Fatal(err) + } + r, _, _ := f.runs.Get(run) + if r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, unit.Ask) || !strings.Contains(r.Said, "30m") { + t.Errorf("the run whose clock ended = %+v, want could not run naming its ask and the bound", r) + } + if len(f.drops) != 1 || f.drops[0] != "node-build-go ace "+unit.Ask { + t.Errorf("drops = %v, want the run's ask dropped of ace", f.drops) + } + q := f.pipeline(p) + for _, art := range []string{"app:program", "app:tools"} { + if s := stepNamed(q, art, "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != r.Said { + t.Errorf("the %s unit step = %+v, want could not run with the run", art, s) + } + } + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask, + Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "passed", Ended: &ended}); err != nil { + t.Fatal(err) + } + if s := stepNamed(f.pipeline(p), "app:program", "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun { + t.Errorf("a step-ended after the run ended moved the step to %s", s.Outcome) + } + if !strings.Contains(strings.Join(f.logged, "\n"), "the step-ended of ask "+unit.Ask+" is ignored: its test run "+run) { + t.Errorf("the ignored step-ended is not journaled: %q", f.logged) + } + // An untaken run whose bound passed while the runner was away ends at the start. + lint := stepNamed(p, "app:program", "lint").Shared[0].Run + delete(f.clock.bound, UntakenPrefix+lint) + if n, err := f.h.TakeUp(); n != 1 || err != nil { + t.Fatalf("take-up ended %d, %v; want the lint run", n, err) + } + if r, _, _ := f.runs.Get(lint); r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "run-untaken-bound") { + t.Errorf("the lint run taken up = %+v", r) + } + }) + t.Run("cap", func(t *testing.T) { + f := newRunFixture() + f.s.Runs.CapBytes, f.fill = 1<<20, 1<<20 + p := f.start(t, 4, 9) + for _, a := range f.asks.ids { + if a == stepNamed(p, "app:program", "unit").Ask { + t.Error("a unit run is asked at the cap") + } + } + q := f.pipeline(p) + for _, art := range []string{"app:program", "app:tools"} { + if s := stepNamed(q, art, "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(s.Why, "1048576 of its 1048576 bytes") { + t.Errorf("the %s unit step at the cap = %+v, want could not run naming the fill", art, s) + } + } + }) + t.Run("age", func(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + unit := stepNamed(p, "app:program", "unit") + run := unit.Shared[0].Run + _ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed, Output: "out-1"}, nil) + f.objects["out-1"] = []byte("ok") + now := agedNow + a := &Ager{Runs: f.runs, RunOutputs: f.objects.Delete, RunsKept: 7 * 24 * time.Hour, Pipelines: agedDeleter{f.store}, + Aged: agedState{}, Now: func() time.Time { return now }, + Log: func(from uint64, limit int) ([]LoggedEntry, error) { + if from > 1 { + return nil, nil + } + return []LoggedEntry{{Sequence: 1, Entry: seat.LogEntry{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools", + What: seat.LogEntryWhatEnded, At: agedNow.Add(-6 * 24 * time.Hour)}}}, nil + }} + a.Age() + if _, _, err := f.runs.Get(run); err != nil { + t.Fatal("a run is aged before test-runs-kept passed") + } + now = now.Add(2 * 24 * time.Hour) + a.Age() + if _, _, err := f.runs.Get(run); err != ErrNoRun || len(f.objects) != 0 { + t.Errorf("the run past test-runs-kept is %v, objects %v; want both deleted", err, f.objects) + } + }) +} + +// agedDeleter is a memStore with the delete the ager makes. +type agedDeleter struct{ memStore } + +func (d agedDeleter) Delete(key string) error { delete(d.memStore, key); return nil } diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 80d88e8a..cc545315 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -55,9 +55,12 @@ func TestManifest(t *testing.T) { {"claims the seat serving every verb of its definition", len(m.Claims) == 1 && m.Claims[0].Name == seat.Seat && equalSets(m.Claims[0].Serves, seat.Verbs())}, // §27, §85, §330, §355, §77: the old state sections. - {"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "toolchains", "findings", "first-views", "consumers"})}, + {"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "test-runs", "toolchains", "findings", "first-views", "consumers"})}, // §535, §647, A-77, A-104: an ask no holder takes is bound by a key with its own time to live. {"bounds is key-ttl", keyTTL["bounds"]}, + // §725, A-469w, A-470: test-runs holds its runs' bound keys, each with its own time to live; 64 MiB its ceiling. + {"test-runs is key-ttl of 64 MiB", keyTTL["test-runs"] && stateMiB["test-runs"] == 64}, + {"the test runs' settings", runSettingsOf(raw)}, // RN25 (§469, §85): an old state section declares its cap, pipelines 256 MiB and kept 128 MiB. {"pipelines 256 MiB", stateMiB[seat.StatePipelines] == 256}, {"kept 128 MiB", stateMiB[seat.StateKept] == 128}, @@ -78,7 +81,8 @@ func TestManifest(t *testing.T) { {"consumes", equalSets(m.Consumes, []string{ "mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended", "operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended", - "node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", "mesh-wants.step-ended", "mesh-catalog.registration-changed", + "node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", + "node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged", // A-186: the seat list its write verbs' callers are checked against, heard as it changes. "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"})}, @@ -175,3 +179,43 @@ func equalSets(a, b []string) bool { slices.Sort(y) return slices.Equal(x, y) } + +// runSettingsOf says whether the manifest holds the test runs' four settings (design 47 G §725; A-469w, A-469x): the cap a +// whole number of at most 64, the three times Go durations, each written into the file its environment names. +func runSettingsOf(raw []byte) bool { + var m struct { + Config map[string]map[string]any `json:"config"` + Resources []struct { + Type string `json:"type"` + Path string `json:"path"` + Content string `json:"content"` + Env map[string]string `json:"env"` + } `json:"resources"` + } + if json.Unmarshal(raw, &m) != nil { + return false + } + want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string"} + if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) { + return false + } + env := map[string]bool{} + for _, r := range m.Resources { + for _, v := range r.Env { + env[v] = true + } + } + for key, kind := range want { + if m.Config[key]["type"] != kind { + return false + } + found := false + for _, r := range m.Resources { + found = found || (r.Type == "file" && r.Content == "${config:"+key+"}" && env[r.Path]) + } + if !found { + return false + } + } + return true +} diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index 9198f2fa..eec137ad 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -41,6 +41,9 @@ "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", + "node-build-go.step-progress", + "node-build-node.step-progress", + "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", @@ -79,6 +82,34 @@ "uses": [ "operator-channel" ], + "config": { + "test-runs-cap-mib": { + "type": "number", + "description": "MiB of the state test-runs at which the runner asks no new test run and ends the step could not run, naming the fill (design 47 G section 725, A-469d); the manifest's max-mib is the ceiling.", + "minimum": 1, + "maximum": 64, + "trusted": false, + "default": 16 + }, + "test-runs-kept": { + "type": "string", + "description": "how long an ended test run and its output object are kept after its end, a Go duration such as 168h (design 47 G section 725, A-469x, A-470a); at most 168h, so the pipeline naming its key is still kept.", + "trusted": false, + "default": "168h" + }, + "test-step-bound": { + "type": "string", + "description": "how long a taken test run's ask may say no progress before the run ends could not run, a Go duration such as 30m, counted from the build seat's ask-taken and again from each step-progress (design 47 G section 725, A-466n, A-469x).", + "trusted": false, + "default": "30m" + }, + "run-untaken-bound": { + "type": "string", + "description": "how long a test run's ask may wait untaken after the run's start before the run ends could not run, a Go duration such as 24h (design 47 G section 725, A-466s, A-469x).", + "trusted": false, + "default": "24h" + } + }, "state": [ { "name": "pipelines", @@ -107,6 +138,12 @@ "history": 1, "key-ttl": true }, + { + "name": "test-runs", + "history": 1, + "max-mib": 64, + "key-ttl": true + }, { "name": "findings", "history": 1 @@ -162,6 +199,38 @@ "place": ".", "unknown": "program" }, + { + "id": "test-runs-cap", + "type": "file", + "path": "${dir:state}/test-runs-cap-mib", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:test-runs-cap-mib}" + }, + { + "id": "test-runs-kept", + "type": "file", + "path": "${dir:state}/test-runs-kept", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:test-runs-kept}" + }, + { + "id": "test-step-bound", + "type": "file", + "path": "${dir:state}/test-step-bound", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:test-step-bound}" + }, + { + "id": "run-untaken-bound", + "type": "file", + "path": "${dir:state}/run-untaken-bound", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:run-untaken-bound}" + }, { "id": "pipeline", "type": "process", @@ -171,7 +240,11 @@ "./mesh-delivery-pipeline" ], "env": { - "MESH_BROKER_FILE": "${dir:state}/broker" + "MESH_BROKER_FILE": "${dir:state}/broker", + "MESH_RUNNER_TEST_RUNS_CAP_FILE": "${dir:state}/test-runs-cap-mib", + "MESH_RUNNER_TEST_RUNS_KEPT_FILE": "${dir:state}/test-runs-kept", + "MESH_RUNNER_TEST_STEP_BOUND_FILE": "${dir:state}/test-step-bound", + "MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound" }, "user": "mesh-delivery-pipeline", "health": {