From 7697ef598857bc767984740775fc60364c619f08 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 12:39:12 +0100 Subject: [PATCH] assigner: a run's bound holds its ask, armed by compare-and-set, and its end comes before the input's delete (A-470n, A-470o, A-470p) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A bound was armed by a delete and a create with no revision, holding nothing, so a late ask-taken or step-progress of a replaced run armed a clock that later ended the run asked next, and an expiry heard after the bound was armed again ended a run that kept saying progress. A bound key now holds its run's ask, published at the key's latest revision with its own time to live, and is deleted at its own revision when the run read again no longer names that ask; a late signal moves nothing and is journaled; an expiry ends the run only while its bound is still absent when read again and the run is taken (bound.) or untaken (untaken.), the untaken expiry's readers said superseded as the key is gone. A step-ended's run end is written before the kept input is deleted; a failed delete asks the event again, whose redelivery finds the run ended and still deletes the input. Builds: design 47 G §720, §719; A-470n, A-470o, A-470p --- .../compositionbounds.go | 114 ++++++++++++++--- .../mesh-delivery-assigner/compositionruns.go | 79 ++++++++---- .../compositionruns_integration_test.go | 22 +++- .../compositionruns_test.go | 115 ++++++++++++++++-- .../compositionstep_integration_test.go | 21 +++- .../cmd/mesh-delivery-assigner/inputs.go | 17 +-- .../cmd/mesh-delivery-assigner/inputs_test.go | 48 ++++++++ .../cmd/mesh-delivery-assigner/main.go | 2 +- 8 files changed, 358 insertions(+), 60 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go index c3776096..b148dbd1 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "slices" "strings" @@ -71,23 +72,65 @@ func (r *compositionRuns) sayWaits(run meshwants.CompositionRun, readers []meshw } } -// untaken is whether the run's ask has not been taken yet: its untaken key stands. -func (r *compositionRuns) untaken(key string) bool { - raw, _, err := r.kv.get(runUntakenPrefix + key) - return err == nil && raw != nil +// untaken is whether the run of ask has not been taken yet: its untaken key stands holding that ask (A-470n). +func (r *compositionRuns) untaken(key, ask string) bool { + held, _, standing, err := r.boundOf(runUntakenPrefix + key) + return err == nil && standing && held == ask } -// heardAskTaken begins a run's clock at its ask's ask-taken: the bound key armed, the untaken key deleted, each reader's wait -// ended ran. An ask no running run names is another asker's. +// boundOf reads a bound key: the ask it holds and its revision; standing false when it is not written. +func (r *compositionRuns) boundOf(boundKey string) (string, uint64, bool, error) { + raw, rev, err := r.kv.get(boundKey) + if err != nil || raw == nil { + return "", 0, false, err + } + var v boundValue + _ = json.Unmarshal(raw, &v) + return v.Ask, rev, true, nil +} + +// armBound writes the bound key prefix+key holding ask, judged from a read of the run, then reads the run again (A-470n): when +// it no longer names ask, or has ended, the bound just written is deleted at its revision, so no bound written from the earlier +// read ends the run after it, and a bound another run wrote since is kept. It answers whether the bound stands for the run. +func (r *compositionRuns) armBound(prefix, key, ask string, ttl time.Duration) (bool, error) { + rev, err := r.kv.arm(prefix+key, ask, ttl) + if err != nil { + return false, err + } + run, _, ok, err := r.read(key) + if err != nil { + return false, err + } + if ok && run.Ended == nil && run.Ask == ask { + return true, nil + } + if err := r.kv.delAt(prefix+key, rev); err != nil && !errors.Is(err, errRunChanged) { + return false, err + } + return false, nil +} + +// heardAskTaken begins a run's clock at its ask's ask-taken: the bound key armed holding the ask, the untaken key of that ask +// deleted, each reader's wait ended ran. An ask no running run names is another asker's, or a late signal of a run replaced or +// ended, which moves nothing and is journaled (A-470n). func (r *compositionRuns) heardAskTaken(e nodebuildgo.AskTaken) error { key, run, err := r.runOfAsk(e.Asker, e.Ask) - if err != nil || key == "" { + if err != nil { return err } - if err := r.kv.arm(runBoundPrefix+key, r.set.stepBound); err != nil { + if key == "" { + r.late("ask-taken", e.Asker, e.Ask) + return nil + } + armed, err := r.armBound(runBoundPrefix, key, e.Ask, r.set.stepBound) + if err != nil { return fmt.Errorf("the bound of the composition run %s is not armed: %w", key, err) } - if r.untaken(key) { + if !armed { + r.late("ask-taken", e.Asker, e.Ask) + return nil + } + if r.untaken(key, e.Ask) { if err := r.clearBounds(key, e.Ask, runUntakenPrefix); err != nil { return err } @@ -96,13 +139,30 @@ func (r *compositionRuns) heardAskTaken(e nodebuildgo.AskTaken) error { return nil } -// heardProgress begins a running run's clock again at each step-progress of its ask (§720: owed progress, not run length). +// heardProgress begins a running run's clock again at each step-progress of its ask (§720: owed progress, not run length); a +// late one moves nothing and is journaled (A-470n). func (r *compositionRuns) heardProgress(e nodebuildgo.StepProgress) error { key, _, err := r.runOfAsk(e.Asker, e.Ask) - if err != nil || key == "" { + if err != nil { return err } - return r.kv.arm(runBoundPrefix+key, r.set.stepBound) + armed := false + if key != "" { + if armed, err = r.armBound(runBoundPrefix, key, e.Ask, r.set.stepBound); err != nil { + return err + } + } + if !armed { + r.late("step-progress", e.Asker, e.Ask) + } + return nil +} + +// late journals a signal of an ask the assigner put whose run no key names running any more (A-470n). +func (r *compositionRuns) late(signal, asker, ask string) { + if asker == runAsker && ask != "" { + r.logf("the %s of the run ask %s is late: no composition run names it running, so it moves nothing (A-470n)", signal, ask) + } } // runOfAsk is the running run of an ask the assigner put, or none. @@ -133,6 +193,27 @@ func (r *compositionRuns) expired(boundKey string) { if err != nil || !ok || run.Ended != nil { return } + // The marker of an expiry holds no value (A-470o): the run ends only while its bound is still absent when read again, and + // the run is taken for bound., untaken for untaken., so an expiry heard after a bound was armed again, or of a run replaced + // since, ends nothing. + other := runUntakenPrefix + if setting == "run-untaken-bound" { + other = runBoundPrefix + } + _, _, armed, err := r.boundOf(boundKey) + if err != nil { + r.logf("the bound %s is not read again, so its expiry ends nothing: %v", boundKey, err) + return + } + held, _, standing, err := r.boundOf(other + key) + if err != nil { + r.logf("the bound %s is not read, so the expiry of %s ends nothing: %v", other+key, boundKey, err) + return + } + if armed || (standing && held == run.Ask) { + r.logf("the expiry of %s ends nothing: the bound is armed again, or the run of ask %s is not where it bounds (A-470o)", boundKey, run.Ask) + return + } said := fmt.Sprintf("%s: its run ask %s said no progress within %s (%s)", run.Machine, run.Ask, bound, setting) if setting == "run-untaken-bound" { said = fmt.Sprintf("%s: its run ask %s was taken by no build node within %s (%s)", run.Machine, run.Ask, bound, setting) @@ -142,14 +223,19 @@ func (r *compositionRuns) expired(boundKey string) { r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err) } } - if _, err := r.endRun(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil); err != nil { + // The untaken key is gone with its expiry, so the run ended by it is said untaken here: its readers' waits end superseded. + if _, err := r.endRunOf(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil, setting == "run-untaken-bound"); err != nil { r.logf("the composition run %s past its bound is not ended: %v", key, err) } } // endRun ends a run (end) and clears its bounds: the untaken key's readers said superseded, both bound keys deleted. func (r *compositionRuns) endRun(key, ask, outcome, said string, verdict map[string]any) (bool, error) { - untaken := r.untaken(key) + return r.endRunOf(key, ask, outcome, said, verdict, r.untaken(key, ask)) +} + +// endRunOf is endRun of a run whose ask is untaken, or not, as its caller judged. +func (r *compositionRuns) endRunOf(key, ask, outcome, said string, verdict map[string]any, untaken bool) (bool, error) { // The readers a rerun carried move with the run's end (§727, A-470d): chosen before the end is written, while the rerun's own // step still waits on it, so it alone says its own end; each mover waits for the end in the key. if moved := r.moved.Load(); moved != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go index 63005ea7..12b019c4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go @@ -134,8 +134,10 @@ type runsKV interface { // of the state is made from a value read before its latest change. delAt(key string, revision uint64) error keys() ([]string, error) - // arm writes a bound key afresh with its own time to live (§720): its expiry is the bus's event. - arm(key string, ttl time.Duration) error + // arm writes a bound key afresh holding the ask of the run it bounds, with its own time to live (§720, A-470n): at the key's + // latest revision, a delete marker counted, read again when another write came between; its expiry is the bus's event. It + // answers the revision written. + arm(key, ask string, ttl time.Duration) (uint64, error) // used is how many bytes the state holds. used() (int64, error) } @@ -143,6 +145,7 @@ type runsKV interface { // busRuns is runsKV over the bus bucket. type busRuns struct { ctx context.Context + js jetstream.JetStream kv jetstream.KeyValue } @@ -173,13 +176,6 @@ func (b busRuns) update(key string, value []byte, revision uint64) (uint64, erro return rev, err } -func (b busRuns) del(key string) error { - if err := b.kv.Delete(b.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { - return err - } - return nil -} - func (b busRuns) delAt(key string, revision uint64) error { err := b.kv.Delete(b.ctx, key, jetstream.LastRevision(revision)) if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) { @@ -200,16 +196,46 @@ func (b busRuns) keys() ([]string, error) { return out, nil } -// arm deletes the key (a delete, never the expiry's purge) and creates it again with its time to live. -func (b busRuns) arm(key string, ttl time.Duration) error { - if err := b.del(key); err != nil { - return err +// boundValue is a bound key's value: the key and the ask of the run it bounds (A-470n). +type boundValue struct { + Bound string `json:"bound"` + Ask string `json:"ask"` +} + +// arm publishes the bound key holding ask with ttl as its own time to live, expecting the key's latest message (a delete +// marker counted), so a write between is read again and never overwritten unseen (A-470n); the earlier message, replaced, +// takes its time to live with it, so the expiry is heard only ttl after the last write. +func (b busRuns) arm(key, ask string, ttl time.Duration) (uint64, error) { + if b.js == nil { + return 0, errors.New("no bus to arm a bound on") } - _, err := b.kv.Create(b.ctx, key, []byte(`{}`), jetstream.KeyTTL(ttl)) - if errors.Is(err, jetstream.ErrKeyExists) { - return nil // armed at once by another write: its clock begins now too + raw, err := json.Marshal(boundValue{Bound: key, Ask: ask}) + if err != nil { + return 0, err + } + subject := "$KV." + b.kv.Bucket() + "." + key + for { + st, err := b.js.Stream(b.ctx, "KV_"+b.kv.Bucket()) + if err != nil { + return 0, err + } + var last uint64 + m, err := st.GetLastMsgForSubject(b.ctx, subject) + switch { + case err == nil: + last = m.Sequence + case !errors.Is(err, jetstream.ErrMsgNotFound): + return 0, err + } + ack, err := b.js.Publish(b.ctx, subject, raw, jetstream.WithMsgTTL(ttl), jetstream.WithExpectLastSequencePerSubject(last)) + if err != nil && strings.Contains(err.Error(), "wrong last sequence") { + continue + } + if err != nil { + return 0, err + } + return ack.Sequence, nil } - return err } func (b busRuns) used() (int64, error) { @@ -369,7 +395,7 @@ func (r *compositionRuns) join(reader meshwants.CompositionReader, candidate, ma return joined{}, err } r.heard() - if run.Ended == nil && r.untaken(key) { + if run.Ended == nil && r.untaken(key, run.Ask) { r.sayWaits(run, []meshwants.CompositionReader{reader}, "") } return joined{key: key, run: run}, nil @@ -395,8 +421,8 @@ func (r *compositionRuns) join(reader meshwants.CompositionReader, candidate, ma return joined{}, err } r.heard() - // An ask waiting in the queue has no clock but run-untaken-bound from its start (A-466s, A-468b, A-470). - if err := r.kv.arm(runUntakenPrefix+key, r.set.untaken); err != nil { + // An ask waiting in the queue has no clock but run-untaken-bound from its start (A-466s, A-468b, A-470, A-470n). + if _, err := r.armBound(runUntakenPrefix, key, ask, r.set.untaken); err != nil { r.logf("the untaken bound of the composition run %s is not armed: %v", key, err) } r.sayWaits(fresh, fresh.Readers, "") @@ -477,8 +503,8 @@ func (r *compositionRuns) deleteRun(key string, gone func(meshwants.CompositionR } } -// clearBounds deletes the bound keys of the run of ask while key holds that run (or none): each at the revision it was read -// at, read again on a conflict, so a bound a newer run of the key armed is never deleted. +// clearBounds deletes the bound keys of the run of ask while key holds that run (or none) and the bound holds that ask: each at +// the revision it was read at, read again on a conflict, so a bound a newer run of the key armed is never deleted. func (r *compositionRuns) clearBounds(key, ask string, prefixes ...string) error { var errs []error for _, prefix := range prefixes { @@ -491,11 +517,14 @@ func (r *compositionRuns) clearBounds(key, ask string, prefixes ...string) error if ok && run.Ask != ask { break // a newer run holds the key: its bounds are its own } - raw, rev, err := r.kv.get(prefix + key) - if err != nil || raw == nil { + held, rev, standing, err := r.boundOf(prefix + key) + if err != nil || !standing { errs = append(errs, err) break } + if held != ask { + break // the bound of another ask (A-470n): not this run's to delete + } if err := r.kv.delAt(prefix+key, rev); errors.Is(err, errRunChanged) { continue } else if err != nil { @@ -806,7 +835,7 @@ func (r *compositionRuns) rejoin(reader meshwants.CompositionReader, candidate, } else if err != nil { return joined{}, err } - if err := r.kv.arm(runUntakenPrefix+key, r.set.untaken); err != nil { + if _, err := r.armBound(runUntakenPrefix, key, ask, r.set.untaken); err != nil { r.logf("the untaken bound of the composition run %s is not armed: %v", key, err) } r.sayWaits(fresh, fresh.Readers, "") diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go index 4b293453..584540c8 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go @@ -31,7 +31,8 @@ import ( // warning; a verdict with no `blamed` is judged as before, "the composer names no culprits", and a machine blaming none fails // every step; a step of another candidate starts its own runs. A taken ask that keeps saying progress is not ended by the bound, // one silent past it ends could not run with its ask dropped, a step-ended after that is ignored and journaled, and an ask never -// taken ends at run-untaken-bound. A rerun of a starts one run on the machine it did not pass and moves b; a superseded step leaves +// taken ends at run-untaken-bound; a bound holds its run's ask (A-470n). A rerun of a starts one run on the machine it did not +// pass and moves b; a superseded step leaves // a run running for its other reader; an assigner restarted during a run takes it up by its ask; compositions-drop ends the // readers and deletes the key; at the cap a step ends could not run, naming the fill. func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) { @@ -278,12 +279,29 @@ func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) { } } say("ask-taken", "b1", nodebuildgo.AskTaken{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Node: "b1"}) - for range 3 { // past the 2 s bound, each beat within it + for i := range 3 { // past the 2 s bound, each beat within it select { case e := <-ended: t.Fatalf("a run saying its progress ended: %+v", e) case <-time.After(time.Second): } + if i == 0 { // the bound holds its run's ask, published at the key's latest revision with its own time to live (A-470n) + runs := b.buckets[buckets.CompositionRuns()] + all, _ := runs.Keys(b.ctx) + held := "" + for _, k := range all { + if strings.HasPrefix(k, runBoundPrefix) { + if e, err := runs.Get(b.ctx, k); err == nil { + var v boundValue + _ = json.Unmarshal(e.Value(), &v) + held += v.Ask + } + } + } + if held != n1.ask.ID { + t.Fatalf("the bound keys hold the asks %q, want n1's %s alone", held, n1.ask.ID) + } + } say("step-progress", "b1", nodebuildgo.StepProgress{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Phase: "run"}) } e := ends(1, "the bounds")["step-r2"] diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go index 4bbd6d37..0dad873e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go @@ -26,6 +26,8 @@ type runsMem struct { n uint64 // beforeDel, when set, runs once before the next delete: another write landing between a read and the delete. beforeDel func() + // beforeArm, when set, runs once before the next arm: another write landing between a read and the arm. + beforeArm func() } func (k *runsMem) another() { @@ -97,12 +99,29 @@ func (k *runsMem) keys() ([]string, error) { return out, nil } -func (k *runsMem) arm(key string, ttl time.Duration) error { +func (k *runsMem) arm(key, ask string, ttl time.Duration) (uint64, error) { + k.mu.Lock() + f := k.beforeArm + k.beforeArm = nil + k.mu.Unlock() + if f != nil { + f() + } + raw, _ := json.Marshal(boundValue{Bound: key, Ask: ask}) k.mu.Lock() defer k.mu.Unlock() k.n++ - k.vals[key], k.revs[key], k.ttl[key] = []byte(`{}`), k.n, ttl - return nil + k.vals[key], k.revs[key], k.ttl[key] = raw, k.n, ttl + return k.n, nil +} + +// expire ends a bound key's time to live: the key gone, as the bus's marker leaves it. +func (k *runsMem) expire(key string) { + k.mu.Lock() + defer k.mu.Unlock() + delete(k.vals, key) + delete(k.revs, key) + delete(k.ttl, key) } func (k *runsMem) used() (int64, error) { @@ -213,7 +232,7 @@ func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) { } old := now.Add(-25 * time.Hour) _, _, _ = r.change(runKeyOf("main.1", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &old, "passed"; return true }) - _ = kv.arm(runBoundPrefix+runKeyOf("main.1", "n1"), time.Minute) + _, _ = kv.arm(runBoundPrefix+runKeyOf("main.1", "n1"), "run-main.1", time.Minute) recent := now.Add(-time.Hour) _, _, _ = r.change(runKeyOf("main.2", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &recent, "passed"; return true }) if _, err := r.join(step("ask-x", "a"), "main.9", "n1", "run-9"); err != nil { @@ -272,7 +291,7 @@ func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) { if !c.ended.IsZero() { _, _, _ = r.change(key, func(run *meshwants.CompositionRun) bool { run.Ended, run.Outcome = &c.ended, "failed"; return true }) } - _ = kv.arm(runBoundPrefix+key, time.Minute) + _, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute) kv.beforeDel = func() { // a rerun replaces the key and arms its untaken bound for { raw, rev, _ := kv.get(key) @@ -287,7 +306,7 @@ func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) { break } } - _ = kv.arm(runUntakenPrefix+key, time.Hour) + _, _ = kv.arm(runUntakenPrefix+key, "run-2", time.Hour) } if err := c.act(r, key); err != nil { t.Fatal(err) @@ -388,6 +407,7 @@ func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) { if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil || kv.revs[runBoundPrefix+key] == before { t.Fatalf("a step-progress does not arm the bound again: %v", err) } + kv.expire(runBoundPrefix + key) r.expired(runBoundPrefix + key) run, _, _, _ := r.read(key) if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 said no progress within 30m0s (composition-step-bound)" || @@ -411,10 +431,11 @@ func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) { } }) t.Run("untaken", func(t *testing.T) { - _, r, saids, drops, _ := setup(t) + kv, r, saids, drops, _ := setup(t) if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil { t.Fatal(err) } + kv.expire(runUntakenPrefix + key) r.expired(runUntakenPrefix + key) run, _, _, _ := r.read(key) if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 was taken by no build node within 24h0m0s (run-untaken-bound)" || @@ -426,6 +447,86 @@ func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) { t.Errorf("an ended run is dropped again: %v", *drops) } }) + // A-470n, A-470o: a bound holds its run's ask, armed for it alone; a late signal of a run replaced moves nothing and is + // journaled; an expiry ends the run only while its bound is still absent and the run taken (bound.) or untaken (untaken.). + rerun := func(kv *runsMem, r *compositionRuns) func() { // a rerun replaces run-1 by run-2, its bounds gone, run-2's untaken armed + return func() { + _, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool { c.Ask = "run-2"; return true }) + kv.expire(runBoundPrefix + key) + _, _ = kv.arm(runUntakenPrefix+key, "run-2", time.Hour) + } + } + taken := func(t *testing.T, r *compositionRuns) { + if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: runAsker, Node: "b1"}); err != nil { + t.Fatal(err) + } + } + for _, c := range []struct { + name string + act func(t *testing.T, kv *runsMem, r *compositionRuns) + ended bool + bound string // the ask the bound key holds, "" for none + untaken string + journal string + }{ + {"a late ask-taken of a run replaced as it is heard", func(t *testing.T, kv *runsMem, r *compositionRuns) { + kv.beforeArm = rerun(kv, r) + taken(t, r) + }, false, "", "run-2", "the ask-taken of the run ask run-1 is late"}, + {"a late step-progress of a run replaced as it is heard", func(t *testing.T, kv *runsMem, r *compositionRuns) { + taken(t, r) + kv.beforeArm = rerun(kv, r) + if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil { + t.Fatal(err) + } + kv.expire(runBoundPrefix + key) // a bound left standing would end run-2 here + r.expired(runBoundPrefix + key) + }, false, "", "run-2", "the step-progress of the run ask run-1 is late"}, + {"a step-progress of a run gone", func(t *testing.T, kv *runsMem, r *compositionRuns) { + taken(t, r) + rerun(kv, r)() + if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil { + t.Fatal(err) + } + }, false, "", "run-2", "the step-progress of the run ask run-1 is late"}, + {"an expiry heard after the bound was armed again", func(t *testing.T, kv *runsMem, r *compositionRuns) { + taken(t, r) + r.expired(runBoundPrefix + key) + }, false, "run-1", "", "the expiry of " + runBoundPrefix + key + " ends nothing"}, + {"the bound's expiry of a run not taken", func(t *testing.T, kv *runsMem, r *compositionRuns) { + _, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute) + kv.expire(runBoundPrefix + key) + r.expired(runBoundPrefix + key) + }, false, "", "run-1", "the expiry of " + runBoundPrefix + key + " ends nothing"}, + {"the untaken expiry of a run taken", func(t *testing.T, kv *runsMem, r *compositionRuns) { + _, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute) + kv.expire(runUntakenPrefix + key) + r.expired(runUntakenPrefix + key) + }, false, "run-1", "", "the expiry of " + runUntakenPrefix + key + " ends nothing"}, + {"the ask-taken of the run's own ask", func(t *testing.T, kv *runsMem, r *compositionRuns) { + taken(t, r) + }, false, "run-1", "", ""}, + } { + t.Run(c.name, func(t *testing.T) { + kv, r, _, _, journal := setup(t) + if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil { + t.Fatal(err) + } + c.act(t, kv, r) + run, _, _, _ := r.read(key) + if (run.Ended != nil) != c.ended { + t.Errorf("the run ended %v (%s), want %v", run.Ended != nil, run.Said, c.ended) + } + for prefix, want := range map[string]string{runBoundPrefix: c.bound, runUntakenPrefix: c.untaken} { + if held, _, _, _ := r.boundOf(prefix + key); held != want { + t.Errorf("%s holds the ask %q, want %q", prefix+key, held, want) + } + } + if c.journal != "" && !slices.ContainsFunc(*journal, func(l string) bool { return strings.Contains(l, c.journal) }) { + t.Errorf("the journal %v does not say %q", *journal, c.journal) + } + }) + } t.Run("taken up at the start", func(t *testing.T) { _, r, _, _, _ := setup(t) for _, c := range []struct{ id, machine string }{{"a", "n1"}, {"a", "n2"}, {"b", "n1"}} { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go index 059569b0..230d16f7 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go @@ -181,9 +181,24 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T) t.Error("the machine away was asked for its input") } } - keys, _ := b.buckets[buckets.Inputs()].Keys(b.ctx) - if len(keys) != 0 { - t.Errorf("the inputs stay kept past their step-ended: %v", keys) + // The run's end comes before the input's delete (A-470o, A-470p): the inputs go after the step's end, heard by their watch. + inputs := b.buckets[buckets.Inputs()] + w, err := inputs.WatchAll(b.ctx) + if err != nil { + t.Fatal(err) + } + defer func() { _ = w.Stop() }() + deadline := time.After(10 * time.Second) + for { + keys, _ := inputs.Keys(b.ctx) + if len(keys) == 0 { + break + } + select { + case <-w.Updates(): + case <-deadline: + t.Fatalf("the inputs stay kept past their step-ended: %v", keys) + } } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs.go index fae279b4..4cb138c9 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs.go @@ -554,9 +554,10 @@ func inputsDrop(k *inputKeeper, ask func(string, any) (json.RawMessage, error)) }} } -// heardStepEnded deletes the input kept for the ask a build seat's step-ended names (design 47 G §354), and hands the end to the -// composition run of that ask (A-240, §719), which writes it in the run's key; one whose input cannot be deleted, or whose run's -// end cannot be written, is given again by the bus. +// heardStepEnded hands the end a build seat's step-ended names to the composition run of that ask (A-240, §719), which writes it +// in the run's key, and then deletes the input kept for the ask (design 47 G §354): the run's end first, so no other act delays +// it past its bound (A-470o, A-470p). One whose run's end cannot be written, or whose input cannot be deleted, is given again by +// the bus; its redelivery finds the run ended, said as ignored for the run, and still deletes the input. func heardStepEnded(k *inputKeeper, runEnded func(buildstep.StepEnded) error, m jetstream.Msg, logf func(string, ...any)) { var e buildstep.StepEnded if err := json.Unmarshal(m.Data(), &e); err != nil { @@ -564,11 +565,6 @@ func heardStepEnded(k *inputKeeper, runEnded func(buildstep.StepEnded) error, m _ = m.Ack() return } - if err := k.ended(e.Ask); err != nil { - logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err) - _ = m.Nak() - return - } if runEnded != nil { if err := runEnded(e); err != nil { logf("the composition run of the ask %s is not ended, so the bus gives its step-ended again: %v", e.Ask, err) @@ -576,5 +572,10 @@ func heardStepEnded(k *inputKeeper, runEnded func(buildstep.StepEnded) error, m return } } + if err := k.ended(e.Ask); err != nil { + logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err) + _ = m.Nak() + return + } _ = m.Ack() } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs_test.go index 51ca72e5..c71fbc33 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs_test.go @@ -16,6 +16,7 @@ import ( "github.com/nats-io/nats.go" meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/stateobjects" ) @@ -24,6 +25,8 @@ type kvMem struct { mu sync.Mutex vals map[string][]byte revs map[string]uint64 + // delErr, when set, fails the next Delete once. + delErr error } func newKVMem() *kvMem { return &kvMem{vals: map[string][]byte{}, revs: map[string]uint64{}} } @@ -58,6 +61,10 @@ func (k *kvMem) Update(key string, v []byte, rev uint64) error { func (k *kvMem) Delete(key string) error { k.mu.Lock() defer k.mu.Unlock() + if err := k.delErr; err != nil { + k.delErr = nil + return err + } delete(k.vals, key) return nil } @@ -247,6 +254,47 @@ func TestAnInputIsDeletedAtItsStepEndedOrAfterADay(t *testing.T) { if _, err := k.body(d2); err == nil { t.Error("an input a day old is still kept at the next ask") } + // A-470o, A-470p: the step-ended's run end is written before the input is deleted; a delete that fails asks the event again, + // whose redelivery finds the run ended and still deletes the input; a run end not written asks it again with the input kept. + for _, c := range []struct { + name string + delErr error + runErr error + naked bool + runs int + deleted bool + }{ + {name: "the run's end before the input's delete", runs: 1, deleted: true}, + {name: "an input not deleted asks the event again", delErr: errors.New("the bus is away"), naked: true, runs: 2, deleted: true}, + {name: "a run's end not written asks the event again", runErr: errors.New("the run is not read"), naked: true, runs: 1}, + } { + t.Run(c.name, func(t *testing.T) { + k := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now) + d, _ := k.keep("ask/run", []byte(`{"heads":"abc"}`)) + k.vals.KV.(*kvMem).delErr = c.delErr + runs := 0 + runEnded := func(e buildstep.StepEnded) error { + runs++ + if _, err := k.body(d); err != nil { + t.Errorf("the run's end is written after the input's delete (%d)", runs) + } + return c.runErr + } + raw, _ := json.Marshal(buildstep.StepEnded{Ask: "ask/run"}) + m := &heardMsg{data: raw, subject: "step-ended"} + heardStepEnded(k, runEnded, m, t.Logf) + if m.naked != c.naked || m.acked == c.naked { + t.Fatalf("acked %v, naked %v, want naked %v", m.acked, m.naked, c.naked) + } + if m.naked && c.delErr != nil { // the redelivery + m = &heardMsg{data: raw, subject: "step-ended"} + heardStepEnded(k, func(e buildstep.StepEnded) error { runs++; return nil }, m, t.Logf) + } + if _, err := k.body(d); runs != c.runs || (err != nil) != c.deleted { + t.Errorf("the run's end heard %d times (want %d), the input deleted %v (want %v)", runs, c.runs, err != nil, c.deleted) + } + }) + } } // Design 47 G A-465: an input the assigner ends itself (the step-ended of its ask, the sweep of a day-old key) ends its object diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index f9ef57fd..a7c86317 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -538,7 +538,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, if err != nil { return err } - runs = newCompositionRuns(busRuns{ctx, kv}, set, time.Now, logger.Printf) + runs = newCompositionRuns(busRuns{ctx, js, kv}, set, time.Now, logger.Printf) if js != nil { runs.say = wantsPublisher(ctx, js, thisNode) }