diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go index da948880..22a724d1 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns.go @@ -31,6 +31,7 @@ const TestRunsState = "test-runs" // time to live (A-77, A-470). type testRunsKV struct { ctx context.Context + js jetstream.JetStream kv jetstream.KeyValue } @@ -93,21 +94,70 @@ func (t testRunsKV) deleteBound(key string) error { return nil } -// Bound writes key again with its time to live: deleted, then created with ttl, so its expiry is heard only ttl after -// the last write (a delete is no expiry). -func (t testRunsKV) Bound(key string, ttl time.Duration) error { - if err := t.deleteBound(key); err != nil { - return err +// boundValue is a bound key's value: the key and the ask of the run it bounds (A-470n). +type boundValue struct { + Bound string `json:"bound"` + Ask string `json:"ask"` +} + +// Bound writes key again holding ask, with ttl as its own time to live, published at the key's latest revision (a delete +// marker counted), so a write between is read again and never overwritten unseen (A-470n); the earlier message, replaced, +// takes its time to live with it, so the expiry is heard only ttl after the last write. +func (t testRunsKV) Bound(key, ask string, ttl time.Duration) (uint64, error) { + raw, err := json.Marshal(boundValue{Bound: key, Ask: ask}) + if err != nil { + return 0, err } - _, err := t.kv.Create(t.ctx, key, []byte(`{"bound":"`+key+`"}`), jetstream.KeyTTL(ttl)) - if errors.Is(err, jetstream.ErrKeyExists) { + subject := "$KV." + t.kv.Bucket() + "." + key + for { + var last uint64 + st, err := t.js.Stream(t.ctx, "KV_"+t.kv.Bucket()) + if err != nil { + return 0, err + } + m, err := st.GetLastMsgForSubject(t.ctx, subject) + switch { + case err == nil: + last = m.Sequence + case !errors.Is(err, jetstream.ErrMsgNotFound): + return 0, err + } + ack, err := t.js.Publish(t.ctx, subject, raw, jetstream.WithMsgTTL(ttl), jetstream.WithExpectLastSequencePerSubject(last)) + if err != nil && strings.Contains(err.Error(), "wrong last sequence") { + continue + } + if err != nil { + return 0, err + } + return ack.Sequence, nil + } +} + +// Bounded reads a bound key: the ask it holds and its revision. +func (t testRunsKV) Bounded(key string) (string, uint64, bool, error) { + e, err := t.kv.Get(t.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return "", 0, false, nil + } + if err != nil { + return "", 0, false, err + } + var v boundValue + _ = json.Unmarshal(e.Value(), &v) + return v.Ask, e.Revision(), true, nil +} + +func (t testRunsKV) Unbound(key string) error { return t.deleteBound(key) } + +// UnboundAt deletes a bound key at revision: a later write of it is kept. +func (t testRunsKV) UnboundAt(key string, revision uint64) error { + err := t.kv.Delete(t.ctx, key, jetstream.LastRevision(revision)) + if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence") || errors.Is(err, jetstream.ErrKeyNotFound)) { return nil } return err } -func (t testRunsKV) Unbound(key string) error { return t.deleteBound(key) } - func (t testRunsKV) Has(key string) (bool, error) { _, err := t.kv.Get(t.ctx, key) if errors.Is(err, jetstream.ErrKeyNotFound) { @@ -351,7 +401,7 @@ func openTestRuns(ctx context.Context, js jetstream.JetStream, cred process.Acco if err != nil { return nil, nil, err } - runs := testRunsKV{ctx: ctx, kv: kv} + runs := testRunsKV{ctx: ctx, js: js, kv: kv} objects := &runObjects{ctx: ctx, js: js, module: cred.Module} starter.TestRuns, starter.Clock, starter.Runs = runs, runs, settings starter.RunsFill = runsFill(ctx, js, cred.Module, time.Now) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go index d694bbb2..e686f14f 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go @@ -38,12 +38,18 @@ type RunObjects interface { Delete(name string) error } -// RunClock is the bound keys of the test runs, keys of test-runs with their own time to live. +// RunClock is the bound keys of the test runs, keys of test-runs with their own time to live, each holding the ask of the +// run it bounds (A-470n). type RunClock interface { - // Bound writes key again with ttl as its time to live. - Bound(key string, ttl time.Duration) error + // Bound writes key again holding ask, with ttl as its time to live, by compare-and-set on the key's latest revision + // (retried when another write came between); it answers the revision written. + Bound(key, ask string, ttl time.Duration) (uint64, error) + // Bounded reads key: the ask it holds and its revision; ok false when it is not written. + Bounded(key string) (ask string, revision uint64, ok bool, err error) // Unbound deletes key; a key gone is no fault. Unbound(key string) error + // UnboundAt deletes key at revision; a later write of it (another run's bound) is kept, and a key gone is no fault. + UnboundAt(key string, revision uint64) error // Has says whether key is written. Has(key string) (bool, error) // Keys are test-runs' keys. @@ -106,6 +112,7 @@ func (h *Hearer) RunTaken(ask, node string) error { return fmt.Errorf("the test run %s is not read: %w", run, err) } if r.Ask != ask || r.Ended != nil { + h.logf("the ask-taken of ask %s is late: the test run %s names ask %s (ended %v), so it moves nothing (A-470n)", ask, run, r.Ask, r.Ended != nil) return nil } if r.Node != node { @@ -119,25 +126,31 @@ func (h *Hearer) RunTaken(ask, node string) error { h.runStarted(run, r) break } - if err := h.bound(run, ask); err != nil { + if err := armBound(h.Clock, h.TestRuns, BoundPrefix+run, run, ask, h.runSettings().StepBound); err != nil { return fmt.Errorf("the clock of the test run %s is not begun: %w", run, err) } - return h.Clock.Unbound(UntakenPrefix + run) -} - -// bound writes the clock of the run under ask, judged from a read of the run, then reads the run again: a run ended since, -// or asked again and not taken, has its clock deleted, so no bound written from the earlier read ends the run after it. -func (h *Hearer) bound(run, ask string) error { - if err := h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound); err != nil { + // The untaken bound is deleted only while it holds this ask: a rerun's own untaken bound is kept. + held, rev, ok, err := h.Clock.Bounded(UntakenPrefix + run) + if err != nil || !ok || held != ask { return err } - r, _, err := h.TestRuns.Get(run) + return h.Clock.UnboundAt(UntakenPrefix+run, rev) +} + +// armBound writes the bound key of the run under ask holding that ask, judged from a read of the run, then reads the run +// again (A-470n): when it no longer names ask, or has ended, the bound just written is deleted at its revision, so no +// bound written from the earlier read ends the run after it, and a bound another run wrote since is kept. +func armBound(clock RunClock, runs TestRuns, key, run, ask string, ttl time.Duration) error { + rev, err := clock.Bound(key, ask, ttl) + if err != nil { + return err + } + r, _, err := runs.Get(run) if err != nil && !errors.Is(err, ErrNoRun) { return err } - // A run asked again and taken since keeps the clock: it is its own as well (its ask-taken writes it). - if errors.Is(err, ErrNoRun) || r.Ended != nil || (r.Ask != ask && r.Node == "") { - return h.Clock.Unbound(BoundPrefix + run) + if errors.Is(err, ErrNoRun) || r.Ended != nil || r.Ask != ask { + return clock.UnboundAt(key, rev) } return nil } @@ -151,12 +164,13 @@ func (h *Hearer) RunProgress(ask string) error { } r, _, err := h.TestRuns.Get(run) if errors.Is(err, ErrNoRun) || (err == nil && (r.Ask != ask || r.Ended != nil)) { + h.logf("the step-progress of ask %s is late: the test run %s names ask %q (ended %v), so it moves nothing (A-470n)", ask, run, r.Ask, r.Ended != nil) return nil } if err != nil { return fmt.Errorf("the test run %s is not read: %w", run, err) } - return h.bound(run, ask) + return armBound(h.Clock, h.TestRuns, BoundPrefix+run, run, ask, h.runSettings().StepBound) } // RunExpired takes the expiry of a test run's bound key (§725, §720 *When the clock ends*, A-466s): the run ends `could not @@ -178,6 +192,14 @@ func (h *Hearer) RunExpired(key string) error { if r.Ended != nil || (untaken && r.Node != "") || (!untaken && r.Node == "") { return nil } + // The marker of an expiry holds no value (A-470o): the run ends only while its bound is still absent when read again, + // so an expiry heard after the bound was armed again for the run's own ask ends nothing. + if _, _, armed, err := h.Clock.Bounded(key); err != nil { + return fmt.Errorf("the bound %s is not read again: %w", key, err) + } else if armed { + h.logf("the expiry of %s ends nothing: the bound is armed again (A-470o)", key) + return nil + } s := h.runSettings() why := fmt.Sprintf("the ask %s of %s was taken by %s and said no progress within %s (test-step-bound)", r.Ask, r.Seat, r.Node, s.StepBound) if untaken { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 6a1ce8ca..63caa13f 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -252,6 +252,9 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh } else if err != nil { return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err) } + // The run's end is written as its step-ended is heard, before any finding the step says, so no wait elsewhere (a + // finding's event the bus is slow to take) delays it past its bound (A-470o). + runErr := h.endRunOf(p, *s, ev) if h.Keep != nil { for _, k := range ev.Kept { kept := seat.KeptResult{Digest: k.Digest, Level: k.Level, Package: k.Package, Pipeline: k.Pipeline, LastRead: &now} @@ -282,7 +285,7 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh if err := h.said(p, asked, ended); err != nil { return p, err } - return p, h.endRunOf(p, *s, ev) + return p, runErr } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go index 5287d4ae..494a2cb7 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go @@ -255,7 +255,7 @@ func (s *Starter) started(a Asked, write func() error) error { if untaken <= 0 { untaken = DefaultRunSettings.Untaken } - if err := s.Clock.Bound(UntakenPrefix+a.Run, untaken); err != nil { + if err := armBound(s.Clock, s.TestRuns, UntakenPrefix+a.Run, a.Run, a.ID, untaken); err != nil { s.logf("the untaken bound of the test run %s is not written: %v", a.Run, err) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index cbbd531b..ad9470ad 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -11,23 +11,41 @@ import ( nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" ) -// clockMem is the bound keys of test-runs in memory, each with the time to live it was last written with. +// clockMem is the bound keys of test-runs in memory, each with the time to live it was last written with, the ask it holds +// and its revision. type clockMem struct { runs testRunsMem bound map[string]time.Duration + asks map[string]string + revs map[string]uint64 + rev uint64 // before is called once at the next Bound, a write that comes between its caller's read and its write. before func() } -func (c *clockMem) Bound(key string, ttl time.Duration) error { +func (c *clockMem) Bound(key, ask string, ttl time.Duration) (uint64, error) { if b := c.before; b != nil { c.before = nil b() } - c.bound[key] = ttl + if c.asks == nil { + c.asks, c.revs = map[string]string{}, map[string]uint64{} + } + c.rev++ + c.bound[key], c.asks[key], c.revs[key] = ttl, ask, c.rev + return c.rev, nil +} +func (c *clockMem) Bounded(key string) (string, uint64, bool, error) { + _, ok := c.bound[key] + return c.asks[key], c.revs[key], ok, nil +} +func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil } +func (c *clockMem) UnboundAt(key string, rev uint64) error { + if c.revs[key] == rev { + delete(c.bound, key) + } return nil } -func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil } func (c *clockMem) Has(key string) (bool, error) { _, ok := c.bound[key]; return ok, nil } func (c *clockMem) Keys() ([]string, error) { var out []string @@ -156,6 +174,14 @@ func TestATestRunHasOneClockACapAndAnAge(t *testing.T) { if r, _, _ := f.runs.Get(run); r.Ended != nil { t.Fatal("a run is ended by its untaken bound after it was taken") } + // An expiry heard once the bound is armed again ends nothing (A-470o); the bound holds the run's ask (A-470n). + if err := f.h.RunExpired(BoundPrefix + run); err != nil { + t.Fatal(err) + } + if r, _, _ := f.runs.Get(run); r.Ended != nil || f.clock.asks[BoundPrefix+run] != unit.Ask { + t.Fatalf("an expiry of a bound armed again ended the run %+v, or the bound holds ask %q", r, f.clock.asks[BoundPrefix+run]) + } + delete(f.clock.bound, BoundPrefix+run) // the bus's expiry deletes it if err := f.h.RunExpired(BoundPrefix + run); err != nil { t.Fatal(err) } @@ -506,6 +532,7 @@ func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) { unit := stepNamed(p, "app:program", "unit") _ = f.h.RunTaken(unit.Ask, "ace") f.h.DropRun = func(string, string, string, string) error { asked(f, run); return nil } + delete(f.clock.bound, BoundPrefix+run) // the bus's expiry deletes it _ = f.h.RunExpired(BoundPrefix + run) }}, {"the last reader to leave ends the run in its own write, so a step joining then asks its own", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) { @@ -563,6 +590,50 @@ func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) { } }) } + // A late signal of a replaced run arms no bound and is said in the journal (A-470n). + t.Run("a late ask-taken or step-progress of a replaced run moves nothing", func(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + unit := stepNamed(p, "app:program", "unit") + run := unit.Shared[0].Run + asked(f, run) + delete(f.clock.bound, UntakenPrefix+run) + _ = f.h.RunTaken(unit.Ask, "ace") + _ = f.h.RunProgress(unit.Ask) + late := 0 + for _, l := range f.logged { + if strings.Contains(l, unit.Ask) && strings.Contains(l, "moves nothing") { + late++ + } + } + if _, ok := f.clock.bound[BoundPrefix+run]; ok || late != 2 { + t.Errorf("bounds %v, journal %q; want no clock and both late signals journaled", f.clock.bound, f.logged) + } + }) + // A run's end is written as its step-ended is heard, before the finding the step says (A-470o). + t.Run("the run ends before the step's finding is written", func(t *testing.T) { + f := newRunFixture() + reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required", + Integration: "bypass", Composition: "bypass", Review: "bypass", Approval: "bypass"}}} + f.s.Registration = func(string) (catalog.Registration, bool) { return reg, true } + p := f.start(t, 4, 9) + unit := stepNamed(p, "app:program", "unit") + run := unit.Shared[0].Run + endedFirst := false + f.h.Failures = &Failures{Findings: newHeld(), Author: func(string, string) (string, error) { + r, _, _ := f.runs.Get(run) + endedFirst = r.Ended != nil + return "alice", nil + }} + ended := time.Now() + if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask, + Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", Ended: &ended}); err != nil { + t.Fatal(err) + } + if !endedFirst { + t.Error("the step's finding was written before its run's end") + } + }) // A rerun whose read of the key is changed before its write (a step joining the ended run) writes from the latest read, // carrying that reader, never answering the change as an error nor putting back the earlier readers. t.Run("a rerun writes its run from the latest read", func(t *testing.T) {