diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go index c3776096..b148dbd1 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "slices" "strings" @@ -71,23 +72,65 @@ func (r *compositionRuns) sayWaits(run meshwants.CompositionRun, readers []meshw } } -// untaken is whether the run's ask has not been taken yet: its untaken key stands. -func (r *compositionRuns) untaken(key string) bool { - raw, _, err := r.kv.get(runUntakenPrefix + key) - return err == nil && raw != nil +// untaken is whether the run of ask has not been taken yet: its untaken key stands holding that ask (A-470n). +func (r *compositionRuns) untaken(key, ask string) bool { + held, _, standing, err := r.boundOf(runUntakenPrefix + key) + return err == nil && standing && held == ask } -// heardAskTaken begins a run's clock at its ask's ask-taken: the bound key armed, the untaken key deleted, each reader's wait -// ended ran. An ask no running run names is another asker's. +// boundOf reads a bound key: the ask it holds and its revision; standing false when it is not written. +func (r *compositionRuns) boundOf(boundKey string) (string, uint64, bool, error) { + raw, rev, err := r.kv.get(boundKey) + if err != nil || raw == nil { + return "", 0, false, err + } + var v boundValue + _ = json.Unmarshal(raw, &v) + return v.Ask, rev, true, nil +} + +// armBound writes the bound key prefix+key holding ask, judged from a read of the run, then reads the run again (A-470n): when +// it no longer names ask, or has ended, the bound just written is deleted at its revision, so no bound written from the earlier +// read ends the run after it, and a bound another run wrote since is kept. It answers whether the bound stands for the run. +func (r *compositionRuns) armBound(prefix, key, ask string, ttl time.Duration) (bool, error) { + rev, err := r.kv.arm(prefix+key, ask, ttl) + if err != nil { + return false, err + } + run, _, ok, err := r.read(key) + if err != nil { + return false, err + } + if ok && run.Ended == nil && run.Ask == ask { + return true, nil + } + if err := r.kv.delAt(prefix+key, rev); err != nil && !errors.Is(err, errRunChanged) { + return false, err + } + return false, nil +} + +// heardAskTaken begins a run's clock at its ask's ask-taken: the bound key armed holding the ask, the untaken key of that ask +// deleted, each reader's wait ended ran. An ask no running run names is another asker's, or a late signal of a run replaced or +// ended, which moves nothing and is journaled (A-470n). func (r *compositionRuns) heardAskTaken(e nodebuildgo.AskTaken) error { key, run, err := r.runOfAsk(e.Asker, e.Ask) - if err != nil || key == "" { + if err != nil { return err } - if err := r.kv.arm(runBoundPrefix+key, r.set.stepBound); err != nil { + if key == "" { + r.late("ask-taken", e.Asker, e.Ask) + return nil + } + armed, err := r.armBound(runBoundPrefix, key, e.Ask, r.set.stepBound) + if err != nil { return fmt.Errorf("the bound of the composition run %s is not armed: %w", key, err) } - if r.untaken(key) { + if !armed { + r.late("ask-taken", e.Asker, e.Ask) + return nil + } + if r.untaken(key, e.Ask) { if err := r.clearBounds(key, e.Ask, runUntakenPrefix); err != nil { return err } @@ -96,13 +139,30 @@ func (r *compositionRuns) heardAskTaken(e nodebuildgo.AskTaken) error { return nil } -// heardProgress begins a running run's clock again at each step-progress of its ask (§720: owed progress, not run length). +// heardProgress begins a running run's clock again at each step-progress of its ask (§720: owed progress, not run length); a +// late one moves nothing and is journaled (A-470n). func (r *compositionRuns) heardProgress(e nodebuildgo.StepProgress) error { key, _, err := r.runOfAsk(e.Asker, e.Ask) - if err != nil || key == "" { + if err != nil { return err } - return r.kv.arm(runBoundPrefix+key, r.set.stepBound) + armed := false + if key != "" { + if armed, err = r.armBound(runBoundPrefix, key, e.Ask, r.set.stepBound); err != nil { + return err + } + } + if !armed { + r.late("step-progress", e.Asker, e.Ask) + } + return nil +} + +// late journals a signal of an ask the assigner put whose run no key names running any more (A-470n). +func (r *compositionRuns) late(signal, asker, ask string) { + if asker == runAsker && ask != "" { + r.logf("the %s of the run ask %s is late: no composition run names it running, so it moves nothing (A-470n)", signal, ask) + } } // runOfAsk is the running run of an ask the assigner put, or none. @@ -133,6 +193,27 @@ func (r *compositionRuns) expired(boundKey string) { if err != nil || !ok || run.Ended != nil { return } + // The marker of an expiry holds no value (A-470o): the run ends only while its bound is still absent when read again, and + // the run is taken for bound., untaken for untaken., so an expiry heard after a bound was armed again, or of a run replaced + // since, ends nothing. + other := runUntakenPrefix + if setting == "run-untaken-bound" { + other = runBoundPrefix + } + _, _, armed, err := r.boundOf(boundKey) + if err != nil { + r.logf("the bound %s is not read again, so its expiry ends nothing: %v", boundKey, err) + return + } + held, _, standing, err := r.boundOf(other + key) + if err != nil { + r.logf("the bound %s is not read, so the expiry of %s ends nothing: %v", other+key, boundKey, err) + return + } + if armed || (standing && held == run.Ask) { + r.logf("the expiry of %s ends nothing: the bound is armed again, or the run of ask %s is not where it bounds (A-470o)", boundKey, run.Ask) + return + } said := fmt.Sprintf("%s: its run ask %s said no progress within %s (%s)", run.Machine, run.Ask, bound, setting) if setting == "run-untaken-bound" { said = fmt.Sprintf("%s: its run ask %s was taken by no build node within %s (%s)", run.Machine, run.Ask, bound, setting) @@ -142,14 +223,19 @@ func (r *compositionRuns) expired(boundKey string) { r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err) } } - if _, err := r.endRun(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil); err != nil { + // The untaken key is gone with its expiry, so the run ended by it is said untaken here: its readers' waits end superseded. + if _, err := r.endRunOf(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil, setting == "run-untaken-bound"); err != nil { r.logf("the composition run %s past its bound is not ended: %v", key, err) } } // endRun ends a run (end) and clears its bounds: the untaken key's readers said superseded, both bound keys deleted. func (r *compositionRuns) endRun(key, ask, outcome, said string, verdict map[string]any) (bool, error) { - untaken := r.untaken(key) + return r.endRunOf(key, ask, outcome, said, verdict, r.untaken(key, ask)) +} + +// endRunOf is endRun of a run whose ask is untaken, or not, as its caller judged. +func (r *compositionRuns) endRunOf(key, ask, outcome, said string, verdict map[string]any, untaken bool) (bool, error) { // The readers a rerun carried move with the run's end (§727, A-470d): chosen before the end is written, while the rerun's own // step still waits on it, so it alone says its own end; each mover waits for the end in the key. if moved := r.moved.Load(); moved != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go index 63005ea7..12b019c4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go @@ -134,8 +134,10 @@ type runsKV interface { // of the state is made from a value read before its latest change. delAt(key string, revision uint64) error keys() ([]string, error) - // arm writes a bound key afresh with its own time to live (§720): its expiry is the bus's event. - arm(key string, ttl time.Duration) error + // arm writes a bound key afresh holding the ask of the run it bounds, with its own time to live (§720, A-470n): at the key's + // latest revision, a delete marker counted, read again when another write came between; its expiry is the bus's event. It + // answers the revision written. + arm(key, ask string, ttl time.Duration) (uint64, error) // used is how many bytes the state holds. used() (int64, error) } @@ -143,6 +145,7 @@ type runsKV interface { // busRuns is runsKV over the bus bucket. type busRuns struct { ctx context.Context + js jetstream.JetStream kv jetstream.KeyValue } @@ -173,13 +176,6 @@ func (b busRuns) update(key string, value []byte, revision uint64) (uint64, erro return rev, err } -func (b busRuns) del(key string) error { - if err := b.kv.Delete(b.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { - return err - } - return nil -} - func (b busRuns) delAt(key string, revision uint64) error { err := b.kv.Delete(b.ctx, key, jetstream.LastRevision(revision)) if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) { @@ -200,16 +196,46 @@ func (b busRuns) keys() ([]string, error) { return out, nil } -// arm deletes the key (a delete, never the expiry's purge) and creates it again with its time to live. -func (b busRuns) arm(key string, ttl time.Duration) error { - if err := b.del(key); err != nil { - return err +// boundValue is a bound key's value: the key and the ask of the run it bounds (A-470n). +type boundValue struct { + Bound string `json:"bound"` + Ask string `json:"ask"` +} + +// arm publishes the bound key holding ask with ttl as its own time to live, expecting the key's latest message (a delete +// marker counted), so a write between is read again and never overwritten unseen (A-470n); the earlier message, replaced, +// takes its time to live with it, so the expiry is heard only ttl after the last write. +func (b busRuns) arm(key, ask string, ttl time.Duration) (uint64, error) { + if b.js == nil { + return 0, errors.New("no bus to arm a bound on") } - _, err := b.kv.Create(b.ctx, key, []byte(`{}`), jetstream.KeyTTL(ttl)) - if errors.Is(err, jetstream.ErrKeyExists) { - return nil // armed at once by another write: its clock begins now too + raw, err := json.Marshal(boundValue{Bound: key, Ask: ask}) + if err != nil { + return 0, err + } + subject := "$KV." + b.kv.Bucket() + "." + key + for { + st, err := b.js.Stream(b.ctx, "KV_"+b.kv.Bucket()) + if err != nil { + return 0, err + } + var last uint64 + m, err := st.GetLastMsgForSubject(b.ctx, subject) + switch { + case err == nil: + last = m.Sequence + case !errors.Is(err, jetstream.ErrMsgNotFound): + return 0, err + } + ack, err := b.js.Publish(b.ctx, subject, raw, jetstream.WithMsgTTL(ttl), jetstream.WithExpectLastSequencePerSubject(last)) + if err != nil && strings.Contains(err.Error(), "wrong last sequence") { + continue + } + if err != nil { + return 0, err + } + return ack.Sequence, nil } - return err } func (b busRuns) used() (int64, error) { @@ -369,7 +395,7 @@ func (r *compositionRuns) join(reader meshwants.CompositionReader, candidate, ma return joined{}, err } r.heard() - if run.Ended == nil && r.untaken(key) { + if run.Ended == nil && r.untaken(key, run.Ask) { r.sayWaits(run, []meshwants.CompositionReader{reader}, "") } return joined{key: key, run: run}, nil @@ -395,8 +421,8 @@ func (r *compositionRuns) join(reader meshwants.CompositionReader, candidate, ma return joined{}, err } r.heard() - // An ask waiting in the queue has no clock but run-untaken-bound from its start (A-466s, A-468b, A-470). - if err := r.kv.arm(runUntakenPrefix+key, r.set.untaken); err != nil { + // An ask waiting in the queue has no clock but run-untaken-bound from its start (A-466s, A-468b, A-470, A-470n). + if _, err := r.armBound(runUntakenPrefix, key, ask, r.set.untaken); err != nil { r.logf("the untaken bound of the composition run %s is not armed: %v", key, err) } r.sayWaits(fresh, fresh.Readers, "") @@ -477,8 +503,8 @@ func (r *compositionRuns) deleteRun(key string, gone func(meshwants.CompositionR } } -// clearBounds deletes the bound keys of the run of ask while key holds that run (or none): each at the revision it was read -// at, read again on a conflict, so a bound a newer run of the key armed is never deleted. +// clearBounds deletes the bound keys of the run of ask while key holds that run (or none) and the bound holds that ask: each at +// the revision it was read at, read again on a conflict, so a bound a newer run of the key armed is never deleted. func (r *compositionRuns) clearBounds(key, ask string, prefixes ...string) error { var errs []error for _, prefix := range prefixes { @@ -491,11 +517,14 @@ func (r *compositionRuns) clearBounds(key, ask string, prefixes ...string) error if ok && run.Ask != ask { break // a newer run holds the key: its bounds are its own } - raw, rev, err := r.kv.get(prefix + key) - if err != nil || raw == nil { + held, rev, standing, err := r.boundOf(prefix + key) + if err != nil || !standing { errs = append(errs, err) break } + if held != ask { + break // the bound of another ask (A-470n): not this run's to delete + } if err := r.kv.delAt(prefix+key, rev); errors.Is(err, errRunChanged) { continue } else if err != nil { @@ -806,7 +835,7 @@ func (r *compositionRuns) rejoin(reader meshwants.CompositionReader, candidate, } else if err != nil { return joined{}, err } - if err := r.kv.arm(runUntakenPrefix+key, r.set.untaken); err != nil { + if _, err := r.armBound(runUntakenPrefix, key, ask, r.set.untaken); err != nil { r.logf("the untaken bound of the composition run %s is not armed: %v", key, err) } r.sayWaits(fresh, fresh.Readers, "") diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go index 4b293453..584540c8 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go @@ -31,7 +31,8 @@ import ( // warning; a verdict with no `blamed` is judged as before, "the composer names no culprits", and a machine blaming none fails // every step; a step of another candidate starts its own runs. A taken ask that keeps saying progress is not ended by the bound, // one silent past it ends could not run with its ask dropped, a step-ended after that is ignored and journaled, and an ask never -// taken ends at run-untaken-bound. A rerun of a starts one run on the machine it did not pass and moves b; a superseded step leaves +// taken ends at run-untaken-bound; a bound holds its run's ask (A-470n). A rerun of a starts one run on the machine it did not +// pass and moves b; a superseded step leaves // a run running for its other reader; an assigner restarted during a run takes it up by its ask; compositions-drop ends the // readers and deletes the key; at the cap a step ends could not run, naming the fill. func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) { @@ -278,12 +279,29 @@ func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) { } } say("ask-taken", "b1", nodebuildgo.AskTaken{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Node: "b1"}) - for range 3 { // past the 2 s bound, each beat within it + for i := range 3 { // past the 2 s bound, each beat within it select { case e := <-ended: t.Fatalf("a run saying its progress ended: %+v", e) case <-time.After(time.Second): } + if i == 0 { // the bound holds its run's ask, published at the key's latest revision with its own time to live (A-470n) + runs := b.buckets[buckets.CompositionRuns()] + all, _ := runs.Keys(b.ctx) + held := "" + for _, k := range all { + if strings.HasPrefix(k, runBoundPrefix) { + if e, err := runs.Get(b.ctx, k); err == nil { + var v boundValue + _ = json.Unmarshal(e.Value(), &v) + held += v.Ask + } + } + } + if held != n1.ask.ID { + t.Fatalf("the bound keys hold the asks %q, want n1's %s alone", held, n1.ask.ID) + } + } say("step-progress", "b1", nodebuildgo.StepProgress{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Phase: "run"}) } e := ends(1, "the bounds")["step-r2"] diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go index 4bbd6d37..0dad873e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go @@ -26,6 +26,8 @@ type runsMem struct { n uint64 // beforeDel, when set, runs once before the next delete: another write landing between a read and the delete. beforeDel func() + // beforeArm, when set, runs once before the next arm: another write landing between a read and the arm. + beforeArm func() } func (k *runsMem) another() { @@ -97,12 +99,29 @@ func (k *runsMem) keys() ([]string, error) { return out, nil } -func (k *runsMem) arm(key string, ttl time.Duration) error { +func (k *runsMem) arm(key, ask string, ttl time.Duration) (uint64, error) { + k.mu.Lock() + f := k.beforeArm + k.beforeArm = nil + k.mu.Unlock() + if f != nil { + f() + } + raw, _ := json.Marshal(boundValue{Bound: key, Ask: ask}) k.mu.Lock() defer k.mu.Unlock() k.n++ - k.vals[key], k.revs[key], k.ttl[key] = []byte(`{}`), k.n, ttl - return nil + k.vals[key], k.revs[key], k.ttl[key] = raw, k.n, ttl + return k.n, nil +} + +// expire ends a bound key's time to live: the key gone, as the bus's marker leaves it. +func (k *runsMem) expire(key string) { + k.mu.Lock() + defer k.mu.Unlock() + delete(k.vals, key) + delete(k.revs, key) + delete(k.ttl, key) } func (k *runsMem) used() (int64, error) { @@ -213,7 +232,7 @@ func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) { } old := now.Add(-25 * time.Hour) _, _, _ = r.change(runKeyOf("main.1", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &old, "passed"; return true }) - _ = kv.arm(runBoundPrefix+runKeyOf("main.1", "n1"), time.Minute) + _, _ = kv.arm(runBoundPrefix+runKeyOf("main.1", "n1"), "run-main.1", time.Minute) recent := now.Add(-time.Hour) _, _, _ = r.change(runKeyOf("main.2", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &recent, "passed"; return true }) if _, err := r.join(step("ask-x", "a"), "main.9", "n1", "run-9"); err != nil { @@ -272,7 +291,7 @@ func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) { if !c.ended.IsZero() { _, _, _ = r.change(key, func(run *meshwants.CompositionRun) bool { run.Ended, run.Outcome = &c.ended, "failed"; return true }) } - _ = kv.arm(runBoundPrefix+key, time.Minute) + _, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute) kv.beforeDel = func() { // a rerun replaces the key and arms its untaken bound for { raw, rev, _ := kv.get(key) @@ -287,7 +306,7 @@ func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) { break } } - _ = kv.arm(runUntakenPrefix+key, time.Hour) + _, _ = kv.arm(runUntakenPrefix+key, "run-2", time.Hour) } if err := c.act(r, key); err != nil { t.Fatal(err) @@ -388,6 +407,7 @@ func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) { if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil || kv.revs[runBoundPrefix+key] == before { t.Fatalf("a step-progress does not arm the bound again: %v", err) } + kv.expire(runBoundPrefix + key) r.expired(runBoundPrefix + key) run, _, _, _ := r.read(key) if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 said no progress within 30m0s (composition-step-bound)" || @@ -411,10 +431,11 @@ func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) { } }) t.Run("untaken", func(t *testing.T) { - _, r, saids, drops, _ := setup(t) + kv, r, saids, drops, _ := setup(t) if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil { t.Fatal(err) } + kv.expire(runUntakenPrefix + key) r.expired(runUntakenPrefix + key) run, _, _, _ := r.read(key) if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 was taken by no build node within 24h0m0s (run-untaken-bound)" || @@ -426,6 +447,86 @@ func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) { t.Errorf("an ended run is dropped again: %v", *drops) } }) + // A-470n, A-470o: a bound holds its run's ask, armed for it alone; a late signal of a run replaced moves nothing and is + // journaled; an expiry ends the run only while its bound is still absent and the run taken (bound.) or untaken (untaken.). + rerun := func(kv *runsMem, r *compositionRuns) func() { // a rerun replaces run-1 by run-2, its bounds gone, run-2's untaken armed + return func() { + _, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool { c.Ask = "run-2"; return true }) + kv.expire(runBoundPrefix + key) + _, _ = kv.arm(runUntakenPrefix+key, "run-2", time.Hour) + } + } + taken := func(t *testing.T, r *compositionRuns) { + if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: runAsker, Node: "b1"}); err != nil { + t.Fatal(err) + } + } + for _, c := range []struct { + name string + act func(t *testing.T, kv *runsMem, r *compositionRuns) + ended bool + bound string // the ask the bound key holds, "" for none + untaken string + journal string + }{ + {"a late ask-taken of a run replaced as it is heard", func(t *testing.T, kv *runsMem, r *compositionRuns) { + kv.beforeArm = rerun(kv, r) + taken(t, r) + }, false, "", "run-2", "the ask-taken of the run ask run-1 is late"}, + {"a late step-progress of a run replaced as it is heard", func(t *testing.T, kv *runsMem, r *compositionRuns) { + taken(t, r) + kv.beforeArm = rerun(kv, r) + if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil { + t.Fatal(err) + } + kv.expire(runBoundPrefix + key) // a bound left standing would end run-2 here + r.expired(runBoundPrefix + key) + }, false, "", "run-2", "the step-progress of the run ask run-1 is late"}, + {"a step-progress of a run gone", func(t *testing.T, kv *runsMem, r *compositionRuns) { + taken(t, r) + rerun(kv, r)() + if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil { + t.Fatal(err) + } + }, false, "", "run-2", "the step-progress of the run ask run-1 is late"}, + {"an expiry heard after the bound was armed again", func(t *testing.T, kv *runsMem, r *compositionRuns) { + taken(t, r) + r.expired(runBoundPrefix + key) + }, false, "run-1", "", "the expiry of " + runBoundPrefix + key + " ends nothing"}, + {"the bound's expiry of a run not taken", func(t *testing.T, kv *runsMem, r *compositionRuns) { + _, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute) + kv.expire(runBoundPrefix + key) + r.expired(runBoundPrefix + key) + }, false, "", "run-1", "the expiry of " + runBoundPrefix + key + " ends nothing"}, + {"the untaken expiry of a run taken", func(t *testing.T, kv *runsMem, r *compositionRuns) { + _, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute) + kv.expire(runUntakenPrefix + key) + r.expired(runUntakenPrefix + key) + }, false, "run-1", "", "the expiry of " + runUntakenPrefix + key + " ends nothing"}, + {"the ask-taken of the run's own ask", func(t *testing.T, kv *runsMem, r *compositionRuns) { + taken(t, r) + }, false, "run-1", "", ""}, + } { + t.Run(c.name, func(t *testing.T) { + kv, r, _, _, journal := setup(t) + if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil { + t.Fatal(err) + } + c.act(t, kv, r) + run, _, _, _ := r.read(key) + if (run.Ended != nil) != c.ended { + t.Errorf("the run ended %v (%s), want %v", run.Ended != nil, run.Said, c.ended) + } + for prefix, want := range map[string]string{runBoundPrefix: c.bound, runUntakenPrefix: c.untaken} { + if held, _, _, _ := r.boundOf(prefix + key); held != want { + t.Errorf("%s holds the ask %q, want %q", prefix+key, held, want) + } + } + if c.journal != "" && !slices.ContainsFunc(*journal, func(l string) bool { return strings.Contains(l, c.journal) }) { + t.Errorf("the journal %v does not say %q", *journal, c.journal) + } + }) + } t.Run("taken up at the start", func(t *testing.T) { _, r, _, _, _ := setup(t) for _, c := range []struct{ id, machine string }{{"a", "n1"}, {"a", "n2"}, {"b", "n1"}} { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go index 059569b0..230d16f7 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go @@ -181,9 +181,24 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T) t.Error("the machine away was asked for its input") } } - keys, _ := b.buckets[buckets.Inputs()].Keys(b.ctx) - if len(keys) != 0 { - t.Errorf("the inputs stay kept past their step-ended: %v", keys) + // The run's end comes before the input's delete (A-470o, A-470p): the inputs go after the step's end, heard by their watch. + inputs := b.buckets[buckets.Inputs()] + w, err := inputs.WatchAll(b.ctx) + if err != nil { + t.Fatal(err) + } + defer func() { _ = w.Stop() }() + deadline := time.After(10 * time.Second) + for { + keys, _ := inputs.Keys(b.ctx) + if len(keys) == 0 { + break + } + select { + case <-w.Updates(): + case <-deadline: + t.Fatalf("the inputs stay kept past their step-ended: %v", keys) + } } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs.go index fae279b4..4cb138c9 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs.go @@ -554,9 +554,10 @@ func inputsDrop(k *inputKeeper, ask func(string, any) (json.RawMessage, error)) }} } -// heardStepEnded deletes the input kept for the ask a build seat's step-ended names (design 47 G §354), and hands the end to the -// composition run of that ask (A-240, §719), which writes it in the run's key; one whose input cannot be deleted, or whose run's -// end cannot be written, is given again by the bus. +// heardStepEnded hands the end a build seat's step-ended names to the composition run of that ask (A-240, §719), which writes it +// in the run's key, and then deletes the input kept for the ask (design 47 G §354): the run's end first, so no other act delays +// it past its bound (A-470o, A-470p). One whose run's end cannot be written, or whose input cannot be deleted, is given again by +// the bus; its redelivery finds the run ended, said as ignored for the run, and still deletes the input. func heardStepEnded(k *inputKeeper, runEnded func(buildstep.StepEnded) error, m jetstream.Msg, logf func(string, ...any)) { var e buildstep.StepEnded if err := json.Unmarshal(m.Data(), &e); err != nil { @@ -564,11 +565,6 @@ func heardStepEnded(k *inputKeeper, runEnded func(buildstep.StepEnded) error, m _ = m.Ack() return } - if err := k.ended(e.Ask); err != nil { - logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err) - _ = m.Nak() - return - } if runEnded != nil { if err := runEnded(e); err != nil { logf("the composition run of the ask %s is not ended, so the bus gives its step-ended again: %v", e.Ask, err) @@ -576,5 +572,10 @@ func heardStepEnded(k *inputKeeper, runEnded func(buildstep.StepEnded) error, m return } } + if err := k.ended(e.Ask); err != nil { + logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err) + _ = m.Nak() + return + } _ = m.Ack() } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs_test.go index 51ca72e5..c71fbc33 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs_test.go @@ -16,6 +16,7 @@ import ( "github.com/nats-io/nats.go" meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/stateobjects" ) @@ -24,6 +25,8 @@ type kvMem struct { mu sync.Mutex vals map[string][]byte revs map[string]uint64 + // delErr, when set, fails the next Delete once. + delErr error } func newKVMem() *kvMem { return &kvMem{vals: map[string][]byte{}, revs: map[string]uint64{}} } @@ -58,6 +61,10 @@ func (k *kvMem) Update(key string, v []byte, rev uint64) error { func (k *kvMem) Delete(key string) error { k.mu.Lock() defer k.mu.Unlock() + if err := k.delErr; err != nil { + k.delErr = nil + return err + } delete(k.vals, key) return nil } @@ -247,6 +254,47 @@ func TestAnInputIsDeletedAtItsStepEndedOrAfterADay(t *testing.T) { if _, err := k.body(d2); err == nil { t.Error("an input a day old is still kept at the next ask") } + // A-470o, A-470p: the step-ended's run end is written before the input is deleted; a delete that fails asks the event again, + // whose redelivery finds the run ended and still deletes the input; a run end not written asks it again with the input kept. + for _, c := range []struct { + name string + delErr error + runErr error + naked bool + runs int + deleted bool + }{ + {name: "the run's end before the input's delete", runs: 1, deleted: true}, + {name: "an input not deleted asks the event again", delErr: errors.New("the bus is away"), naked: true, runs: 2, deleted: true}, + {name: "a run's end not written asks the event again", runErr: errors.New("the run is not read"), naked: true, runs: 1}, + } { + t.Run(c.name, func(t *testing.T) { + k := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now) + d, _ := k.keep("ask/run", []byte(`{"heads":"abc"}`)) + k.vals.KV.(*kvMem).delErr = c.delErr + runs := 0 + runEnded := func(e buildstep.StepEnded) error { + runs++ + if _, err := k.body(d); err != nil { + t.Errorf("the run's end is written after the input's delete (%d)", runs) + } + return c.runErr + } + raw, _ := json.Marshal(buildstep.StepEnded{Ask: "ask/run"}) + m := &heardMsg{data: raw, subject: "step-ended"} + heardStepEnded(k, runEnded, m, t.Logf) + if m.naked != c.naked || m.acked == c.naked { + t.Fatalf("acked %v, naked %v, want naked %v", m.acked, m.naked, c.naked) + } + if m.naked && c.delErr != nil { // the redelivery + m = &heardMsg{data: raw, subject: "step-ended"} + heardStepEnded(k, func(e buildstep.StepEnded) error { runs++; return nil }, m, t.Logf) + } + if _, err := k.body(d); runs != c.runs || (err != nil) != c.deleted { + t.Errorf("the run's end heard %d times (want %d), the input deleted %v (want %v)", runs, c.runs, err != nil, c.deleted) + } + }) + } } // Design 47 G A-465: an input the assigner ends itself (the step-ended of its ask, the sweep of a day-old key) ends its object diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index f9ef57fd..a7c86317 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -538,7 +538,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, if err != nil { return err } - runs = newCompositionRuns(busRuns{ctx, kv}, set, time.Now, logger.Printf) + runs = newCompositionRuns(busRuns{ctx, js, kv}, set, time.Now, logger.Printf) if js != nil { runs.say = wantsPublisher(ctx, js, thisNode) }