runner: a run's bound holds its ask, armed by compare-and-set, and its end is written before the step's finding (A-470n, A-470o)
A bound was armed by a delete and a create with no revision, so a late ask-taken or step-progress of a replaced run could arm a clock for the run asked next, an expiry heard after the bound was armed again ended the run, and a step-ended's run end waited behind its finding's event (3 s in test 109, past its bound). A bound key now holds the run's ask and is published at the key's latest revision with its own time to live, read against the run after it is written and deleted at its own revision when the run no longer names that ask; a late signal moves nothing and is journaled; an expiry ends the run only while its bound is still absent when read again; the run's end is written as its step-ended is heard, before the finding. Builds: design 47 G §720, §725, §726; A-470n, A-470o
This commit is contained in:
@@ -31,6 +31,7 @@ const TestRunsState = "test-runs"
|
||||
// time to live (A-77, A-470).
|
||||
type testRunsKV struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
@@ -93,21 +94,70 @@ func (t testRunsKV) deleteBound(key string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Bound writes key again with its time to live: deleted, then created with ttl, so its expiry is heard only ttl after
|
||||
// the last write (a delete is no expiry).
|
||||
func (t testRunsKV) Bound(key string, ttl time.Duration) error {
|
||||
if err := t.deleteBound(key); err != nil {
|
||||
return err
|
||||
// boundValue is a bound key's value: the key and the ask of the run it bounds (A-470n).
|
||||
type boundValue struct {
|
||||
Bound string `json:"bound"`
|
||||
Ask string `json:"ask"`
|
||||
}
|
||||
|
||||
// Bound writes key again holding ask, with ttl as its own time to live, published at the key's latest revision (a delete
|
||||
// marker counted), so a write between is read again and never overwritten unseen (A-470n); the earlier message, replaced,
|
||||
// takes its time to live with it, so the expiry is heard only ttl after the last write.
|
||||
func (t testRunsKV) Bound(key, ask string, ttl time.Duration) (uint64, error) {
|
||||
raw, err := json.Marshal(boundValue{Bound: key, Ask: ask})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
_, err := t.kv.Create(t.ctx, key, []byte(`{"bound":"`+key+`"}`), jetstream.KeyTTL(ttl))
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
subject := "$KV." + t.kv.Bucket() + "." + key
|
||||
for {
|
||||
var last uint64
|
||||
st, err := t.js.Stream(t.ctx, "KV_"+t.kv.Bucket())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
m, err := st.GetLastMsgForSubject(t.ctx, subject)
|
||||
switch {
|
||||
case err == nil:
|
||||
last = m.Sequence
|
||||
case !errors.Is(err, jetstream.ErrMsgNotFound):
|
||||
return 0, err
|
||||
}
|
||||
ack, err := t.js.Publish(t.ctx, subject, raw, jetstream.WithMsgTTL(ttl), jetstream.WithExpectLastSequencePerSubject(last))
|
||||
if err != nil && strings.Contains(err.Error(), "wrong last sequence") {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return ack.Sequence, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Bounded reads a bound key: the ask it holds and its revision.
|
||||
func (t testRunsKV) Bounded(key string) (string, uint64, bool, error) {
|
||||
e, err := t.kv.Get(t.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return "", 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", 0, false, err
|
||||
}
|
||||
var v boundValue
|
||||
_ = json.Unmarshal(e.Value(), &v)
|
||||
return v.Ask, e.Revision(), true, nil
|
||||
}
|
||||
|
||||
func (t testRunsKV) Unbound(key string) error { return t.deleteBound(key) }
|
||||
|
||||
// UnboundAt deletes a bound key at revision: a later write of it is kept.
|
||||
func (t testRunsKV) UnboundAt(key string, revision uint64) error {
|
||||
err := t.kv.Delete(t.ctx, key, jetstream.LastRevision(revision))
|
||||
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence") || errors.Is(err, jetstream.ErrKeyNotFound)) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (t testRunsKV) Unbound(key string) error { return t.deleteBound(key) }
|
||||
|
||||
func (t testRunsKV) Has(key string) (bool, error) {
|
||||
_, err := t.kv.Get(t.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
@@ -351,7 +401,7 @@ func openTestRuns(ctx context.Context, js jetstream.JetStream, cred process.Acco
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
runs := testRunsKV{ctx: ctx, kv: kv}
|
||||
runs := testRunsKV{ctx: ctx, js: js, kv: kv}
|
||||
objects := &runObjects{ctx: ctx, js: js, module: cred.Module}
|
||||
starter.TestRuns, starter.Clock, starter.Runs = runs, runs, settings
|
||||
starter.RunsFill = runsFill(ctx, js, cred.Module, time.Now)
|
||||
|
||||
@@ -38,12 +38,18 @@ type RunObjects interface {
|
||||
Delete(name string) error
|
||||
}
|
||||
|
||||
// RunClock is the bound keys of the test runs, keys of test-runs with their own time to live.
|
||||
// RunClock is the bound keys of the test runs, keys of test-runs with their own time to live, each holding the ask of the
|
||||
// run it bounds (A-470n).
|
||||
type RunClock interface {
|
||||
// Bound writes key again with ttl as its time to live.
|
||||
Bound(key string, ttl time.Duration) error
|
||||
// Bound writes key again holding ask, with ttl as its time to live, by compare-and-set on the key's latest revision
|
||||
// (retried when another write came between); it answers the revision written.
|
||||
Bound(key, ask string, ttl time.Duration) (uint64, error)
|
||||
// Bounded reads key: the ask it holds and its revision; ok false when it is not written.
|
||||
Bounded(key string) (ask string, revision uint64, ok bool, err error)
|
||||
// Unbound deletes key; a key gone is no fault.
|
||||
Unbound(key string) error
|
||||
// UnboundAt deletes key at revision; a later write of it (another run's bound) is kept, and a key gone is no fault.
|
||||
UnboundAt(key string, revision uint64) error
|
||||
// Has says whether key is written.
|
||||
Has(key string) (bool, error)
|
||||
// Keys are test-runs' keys.
|
||||
@@ -106,6 +112,7 @@ func (h *Hearer) RunTaken(ask, node string) error {
|
||||
return fmt.Errorf("the test run %s is not read: %w", run, err)
|
||||
}
|
||||
if r.Ask != ask || r.Ended != nil {
|
||||
h.logf("the ask-taken of ask %s is late: the test run %s names ask %s (ended %v), so it moves nothing (A-470n)", ask, run, r.Ask, r.Ended != nil)
|
||||
return nil
|
||||
}
|
||||
if r.Node != node {
|
||||
@@ -119,25 +126,31 @@ func (h *Hearer) RunTaken(ask, node string) error {
|
||||
h.runStarted(run, r)
|
||||
break
|
||||
}
|
||||
if err := h.bound(run, ask); err != nil {
|
||||
if err := armBound(h.Clock, h.TestRuns, BoundPrefix+run, run, ask, h.runSettings().StepBound); err != nil {
|
||||
return fmt.Errorf("the clock of the test run %s is not begun: %w", run, err)
|
||||
}
|
||||
return h.Clock.Unbound(UntakenPrefix + run)
|
||||
}
|
||||
|
||||
// bound writes the clock of the run under ask, judged from a read of the run, then reads the run again: a run ended since,
|
||||
// or asked again and not taken, has its clock deleted, so no bound written from the earlier read ends the run after it.
|
||||
func (h *Hearer) bound(run, ask string) error {
|
||||
if err := h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound); err != nil {
|
||||
// The untaken bound is deleted only while it holds this ask: a rerun's own untaken bound is kept.
|
||||
held, rev, ok, err := h.Clock.Bounded(UntakenPrefix + run)
|
||||
if err != nil || !ok || held != ask {
|
||||
return err
|
||||
}
|
||||
r, _, err := h.TestRuns.Get(run)
|
||||
return h.Clock.UnboundAt(UntakenPrefix+run, rev)
|
||||
}
|
||||
|
||||
// armBound writes the bound key of the run under ask holding that ask, judged from a read of the run, then reads the run
|
||||
// again (A-470n): when it no longer names ask, or has ended, the bound just written is deleted at its revision, so no
|
||||
// bound written from the earlier read ends the run after it, and a bound another run wrote since is kept.
|
||||
func armBound(clock RunClock, runs TestRuns, key, run, ask string, ttl time.Duration) error {
|
||||
rev, err := clock.Bound(key, ask, ttl)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r, _, err := runs.Get(run)
|
||||
if err != nil && !errors.Is(err, ErrNoRun) {
|
||||
return err
|
||||
}
|
||||
// A run asked again and taken since keeps the clock: it is its own as well (its ask-taken writes it).
|
||||
if errors.Is(err, ErrNoRun) || r.Ended != nil || (r.Ask != ask && r.Node == "") {
|
||||
return h.Clock.Unbound(BoundPrefix + run)
|
||||
if errors.Is(err, ErrNoRun) || r.Ended != nil || r.Ask != ask {
|
||||
return clock.UnboundAt(key, rev)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -151,12 +164,13 @@ func (h *Hearer) RunProgress(ask string) error {
|
||||
}
|
||||
r, _, err := h.TestRuns.Get(run)
|
||||
if errors.Is(err, ErrNoRun) || (err == nil && (r.Ask != ask || r.Ended != nil)) {
|
||||
h.logf("the step-progress of ask %s is late: the test run %s names ask %q (ended %v), so it moves nothing (A-470n)", ask, run, r.Ask, r.Ended != nil)
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the test run %s is not read: %w", run, err)
|
||||
}
|
||||
return h.bound(run, ask)
|
||||
return armBound(h.Clock, h.TestRuns, BoundPrefix+run, run, ask, h.runSettings().StepBound)
|
||||
}
|
||||
|
||||
// RunExpired takes the expiry of a test run's bound key (§725, §720 *When the clock ends*, A-466s): the run ends `could not
|
||||
@@ -178,6 +192,14 @@ func (h *Hearer) RunExpired(key string) error {
|
||||
if r.Ended != nil || (untaken && r.Node != "") || (!untaken && r.Node == "") {
|
||||
return nil
|
||||
}
|
||||
// The marker of an expiry holds no value (A-470o): the run ends only while its bound is still absent when read again,
|
||||
// so an expiry heard after the bound was armed again for the run's own ask ends nothing.
|
||||
if _, _, armed, err := h.Clock.Bounded(key); err != nil {
|
||||
return fmt.Errorf("the bound %s is not read again: %w", key, err)
|
||||
} else if armed {
|
||||
h.logf("the expiry of %s ends nothing: the bound is armed again (A-470o)", key)
|
||||
return nil
|
||||
}
|
||||
s := h.runSettings()
|
||||
why := fmt.Sprintf("the ask %s of %s was taken by %s and said no progress within %s (test-step-bound)", r.Ask, r.Seat, r.Node, s.StepBound)
|
||||
if untaken {
|
||||
|
||||
@@ -252,6 +252,9 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh
|
||||
} else if err != nil {
|
||||
return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err)
|
||||
}
|
||||
// The run's end is written as its step-ended is heard, before any finding the step says, so no wait elsewhere (a
|
||||
// finding's event the bus is slow to take) delays it past its bound (A-470o).
|
||||
runErr := h.endRunOf(p, *s, ev)
|
||||
if h.Keep != nil {
|
||||
for _, k := range ev.Kept {
|
||||
kept := seat.KeptResult{Digest: k.Digest, Level: k.Level, Package: k.Package, Pipeline: k.Pipeline, LastRead: &now}
|
||||
@@ -282,7 +285,7 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh
|
||||
if err := h.said(p, asked, ended); err != nil {
|
||||
return p, err
|
||||
}
|
||||
return p, h.endRunOf(p, *s, ev)
|
||||
return p, runErr
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -255,7 +255,7 @@ func (s *Starter) started(a Asked, write func() error) error {
|
||||
if untaken <= 0 {
|
||||
untaken = DefaultRunSettings.Untaken
|
||||
}
|
||||
if err := s.Clock.Bound(UntakenPrefix+a.Run, untaken); err != nil {
|
||||
if err := armBound(s.Clock, s.TestRuns, UntakenPrefix+a.Run, a.Run, a.ID, untaken); err != nil {
|
||||
s.logf("the untaken bound of the test run %s is not written: %v", a.Run, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,23 +11,41 @@ import (
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
)
|
||||
|
||||
// clockMem is the bound keys of test-runs in memory, each with the time to live it was last written with.
|
||||
// clockMem is the bound keys of test-runs in memory, each with the time to live it was last written with, the ask it holds
|
||||
// and its revision.
|
||||
type clockMem struct {
|
||||
runs testRunsMem
|
||||
bound map[string]time.Duration
|
||||
asks map[string]string
|
||||
revs map[string]uint64
|
||||
rev uint64
|
||||
// before is called once at the next Bound, a write that comes between its caller's read and its write.
|
||||
before func()
|
||||
}
|
||||
|
||||
func (c *clockMem) Bound(key string, ttl time.Duration) error {
|
||||
func (c *clockMem) Bound(key, ask string, ttl time.Duration) (uint64, error) {
|
||||
if b := c.before; b != nil {
|
||||
c.before = nil
|
||||
b()
|
||||
}
|
||||
c.bound[key] = ttl
|
||||
if c.asks == nil {
|
||||
c.asks, c.revs = map[string]string{}, map[string]uint64{}
|
||||
}
|
||||
c.rev++
|
||||
c.bound[key], c.asks[key], c.revs[key] = ttl, ask, c.rev
|
||||
return c.rev, nil
|
||||
}
|
||||
func (c *clockMem) Bounded(key string) (string, uint64, bool, error) {
|
||||
_, ok := c.bound[key]
|
||||
return c.asks[key], c.revs[key], ok, nil
|
||||
}
|
||||
func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil }
|
||||
func (c *clockMem) UnboundAt(key string, rev uint64) error {
|
||||
if c.revs[key] == rev {
|
||||
delete(c.bound, key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil }
|
||||
func (c *clockMem) Has(key string) (bool, error) { _, ok := c.bound[key]; return ok, nil }
|
||||
func (c *clockMem) Keys() ([]string, error) {
|
||||
var out []string
|
||||
@@ -156,6 +174,14 @@ func TestATestRunHasOneClockACapAndAnAge(t *testing.T) {
|
||||
if r, _, _ := f.runs.Get(run); r.Ended != nil {
|
||||
t.Fatal("a run is ended by its untaken bound after it was taken")
|
||||
}
|
||||
// An expiry heard once the bound is armed again ends nothing (A-470o); the bound holds the run's ask (A-470n).
|
||||
if err := f.h.RunExpired(BoundPrefix + run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r, _, _ := f.runs.Get(run); r.Ended != nil || f.clock.asks[BoundPrefix+run] != unit.Ask {
|
||||
t.Fatalf("an expiry of a bound armed again ended the run %+v, or the bound holds ask %q", r, f.clock.asks[BoundPrefix+run])
|
||||
}
|
||||
delete(f.clock.bound, BoundPrefix+run) // the bus's expiry deletes it
|
||||
if err := f.h.RunExpired(BoundPrefix + run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -506,6 +532,7 @@ func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) {
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
_ = f.h.RunTaken(unit.Ask, "ace")
|
||||
f.h.DropRun = func(string, string, string, string) error { asked(f, run); return nil }
|
||||
delete(f.clock.bound, BoundPrefix+run) // the bus's expiry deletes it
|
||||
_ = f.h.RunExpired(BoundPrefix + run)
|
||||
}},
|
||||
{"the last reader to leave ends the run in its own write, so a step joining then asks its own", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
|
||||
@@ -563,6 +590,50 @@ func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
// A late signal of a replaced run arms no bound and is said in the journal (A-470n).
|
||||
t.Run("a late ask-taken or step-progress of a replaced run moves nothing", func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
p := f.start(t, 4, 9)
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
run := unit.Shared[0].Run
|
||||
asked(f, run)
|
||||
delete(f.clock.bound, UntakenPrefix+run)
|
||||
_ = f.h.RunTaken(unit.Ask, "ace")
|
||||
_ = f.h.RunProgress(unit.Ask)
|
||||
late := 0
|
||||
for _, l := range f.logged {
|
||||
if strings.Contains(l, unit.Ask) && strings.Contains(l, "moves nothing") {
|
||||
late++
|
||||
}
|
||||
}
|
||||
if _, ok := f.clock.bound[BoundPrefix+run]; ok || late != 2 {
|
||||
t.Errorf("bounds %v, journal %q; want no clock and both late signals journaled", f.clock.bound, f.logged)
|
||||
}
|
||||
})
|
||||
// A run's end is written as its step-ended is heard, before the finding the step says (A-470o).
|
||||
t.Run("the run ends before the step's finding is written", func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required",
|
||||
Integration: "bypass", Composition: "bypass", Review: "bypass", Approval: "bypass"}}}
|
||||
f.s.Registration = func(string) (catalog.Registration, bool) { return reg, true }
|
||||
p := f.start(t, 4, 9)
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
run := unit.Shared[0].Run
|
||||
endedFirst := false
|
||||
f.h.Failures = &Failures{Findings: newHeld(), Author: func(string, string) (string, error) {
|
||||
r, _, _ := f.runs.Get(run)
|
||||
endedFirst = r.Ended != nil
|
||||
return "alice", nil
|
||||
}}
|
||||
ended := time.Now()
|
||||
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
|
||||
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", Ended: &ended}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !endedFirst {
|
||||
t.Error("the step's finding was written before its run's end")
|
||||
}
|
||||
})
|
||||
// A rerun whose read of the key is changed before its write (a step joining the ended run) writes from the latest read,
|
||||
// carrying that reader, never answering the change as an error nor putting back the earlier readers.
|
||||
t.Run("a rerun writes its run from the latest read", func(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user