assigner: no delete of composition-runs lands on a value newer than the one it judged

Every value write of the state was already a compare-and-set from the latest revision, retried on
conflict; the deletes were not. The sweep and the drop deleted a run's key whatever it held, so a
rerun's run replacing the key between the read and the delete was lost; the end of a run, the
ask-taken and a rerun deleted bound keys a newer run of the key may have armed. Each delete is now
at the revision just read, while the key holds the run it was judged from, judged again on a
conflict (the kind the runner's test 109 met in test-runs).

Builds: design 47 G §719 (readers added and removed by compare-and-set; the key deleted after composition-runs-kept), §720 (the bound keys)
This commit is contained in:
2026-11-17 12:23:38 +01:00
parent a2f4e8b499
commit 00e1588ef9
4 changed files with 145 additions and 16 deletions
@@ -88,7 +88,7 @@ func (r *compositionRuns) heardAskTaken(e nodebuildgo.AskTaken) error {
return fmt.Errorf("the bound of the composition run %s is not armed: %w", key, err)
}
if r.untaken(key) {
if err := r.kv.del(runUntakenPrefix + key); err != nil {
if err := r.clearBounds(key, e.Ask, runUntakenPrefix); err != nil {
return err
}
r.sayWaits(run, run.Readers, "ran")
@@ -166,10 +166,8 @@ func (r *compositionRuns) endRun(key, ask, outcome, said string, verdict map[str
r.sayWaits(run, run.Readers, "superseded")
}
}
for _, k := range []string{runBoundPrefix + key, runUntakenPrefix + key} {
if err := r.kv.del(k); err != nil {
r.logf("the bound key %s of the ended composition run is not deleted: %v", k, err)
}
if err := r.clearBounds(key, ask, runBoundPrefix, runUntakenPrefix); err != nil {
r.logf("the bound keys of the ended composition run %s are not deleted: %v", key, err)
}
return true, nil
}
@@ -130,7 +130,9 @@ type runsKV interface {
// create writes a key that does not stand, update one at its revision: errRunChanged when it does not stand so.
create(key string, value []byte) (uint64, error)
update(key string, value []byte, revision uint64) (uint64, error)
del(key string) error
// delAt deletes a key at its revision: errRunChanged when it no longer stands so, nothing for a key not there. No write
// of the state is made from a value read before its latest change.
delAt(key string, revision uint64) error
keys() ([]string, error)
// arm writes a bound key afresh with its own time to live (§720): its expiry is the bus's event.
arm(key string, ttl time.Duration) error
@@ -178,6 +180,14 @@ func (b busRuns) del(key string) error {
return nil
}
func (b busRuns) delAt(key string, revision uint64) error {
err := b.kv.Delete(b.ctx, key, jetstream.LastRevision(revision))
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
return errRunChanged
}
return err
}
func (b busRuns) keys() ([]string, error) {
l, err := b.kv.ListKeys(b.ctx)
if err != nil {
@@ -438,7 +448,60 @@ func (r *compositionRuns) sweep() error {
var errs []error
for _, e := range ended {
if r.now().Sub(e.at) >= r.set.kept {
errs = append(errs, r.kv.del(e.key), r.kv.del(runBoundPrefix+e.key), r.kv.del(runUntakenPrefix+e.key))
errs = append(errs, r.deleteRun(e.key, func(run meshwants.CompositionRun) bool {
return run.Ended != nil && r.now().Sub(*run.Ended) >= r.set.kept
}))
}
}
return errors.Join(errs...)
}
// deleteRun deletes a run's key with its bound keys while the value it holds is still one gone judges gone, each delete at the
// revision just read and judged again on a conflict: a run that replaced the key meanwhile, and its bounds, stand.
func (r *compositionRuns) deleteRun(key string, gone func(meshwants.CompositionRun) bool) error {
for {
run, rev, ok, err := r.read(key)
if err != nil || !ok || !gone(run) {
return err
}
if err := r.clearBounds(key, run.Ask, runBoundPrefix, runUntakenPrefix); err != nil {
return err
}
if err := r.kv.delAt(key, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return err
}
r.heard()
return nil
}
}
// clearBounds deletes the bound keys of the run of ask while key holds that run (or none): each at the revision it was read
// at, read again on a conflict, so a bound a newer run of the key armed is never deleted.
func (r *compositionRuns) clearBounds(key, ask string, prefixes ...string) error {
var errs []error
for _, prefix := range prefixes {
for {
run, _, ok, err := r.read(key)
if err != nil {
errs = append(errs, err)
break
}
if ok && run.Ask != ask {
break // a newer run holds the key: its bounds are its own
}
raw, rev, err := r.kv.get(prefix + key)
if err != nil || raw == nil {
errs = append(errs, err)
break
}
if err := r.kv.delAt(prefix+key, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
errs = append(errs, err)
}
break
}
}
return errors.Join(errs...)
@@ -735,14 +798,14 @@ func (r *compositionRuns) rejoin(reader meshwants.CompositionReader, candidate,
fresh.Readers = append(fresh.Readers, c)
}
fresh.Readers = append(fresh.Readers, reader)
if err := r.clearBounds(key, run.Ask, runBoundPrefix, runUntakenPrefix); err != nil { // the former run's bounds go with it
r.logf("the bound keys of the former composition run %s are not deleted: %v", key, err)
}
if err := r.write(key, fresh, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return joined{}, err
}
for _, k := range []string{runBoundPrefix + key, runUntakenPrefix + key} {
_ = r.kv.del(k) // the former run's bounds go with it
}
if err := r.kv.arm(runUntakenPrefix+key, r.set.untaken); err != nil {
r.logf("the untaken bound of the composition run %s is not armed: %v", key, err)
}
@@ -24,6 +24,18 @@ type runsMem struct {
ttl map[string]time.Duration
bytes int64
n uint64
// beforeDel, when set, runs once before the next delete: another write landing between a read and the delete.
beforeDel func()
}
func (k *runsMem) another() {
k.mu.Lock()
f := k.beforeDel
k.beforeDel = nil
k.mu.Unlock()
if f != nil {
f()
}
}
func newRunsMem() *runsMem {
@@ -58,9 +70,16 @@ func (k *runsMem) update(key string, v []byte, rev uint64) (uint64, error) {
return k.n, nil
}
func (k *runsMem) del(key string) error {
func (k *runsMem) delAt(key string, rev uint64) error {
k.another()
k.mu.Lock()
defer k.mu.Unlock()
if _, ok := k.vals[key]; !ok {
return nil
}
if k.revs[key] != rev {
return errRunChanged
}
delete(k.vals, key)
delete(k.revs, key)
delete(k.ttl, key)
@@ -230,6 +249,57 @@ func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) {
t.Errorf("a key the watch saw deleted is still listed: %v", keys)
}
})
// No delete lands on a value newer than the one it was judged from (the kind test 109 met in test-runs): a rerun's run
// replacing the key, and its untaken bound, stand.
for _, c := range []struct {
name string
ended time.Time
act func(r *compositionRuns, key string) error
}{
{"the sweep of a run replaced meanwhile", now.Add(-25 * time.Hour), func(r *compositionRuns, _ string) error { return r.sweep() }},
{"the end of a run replaced meanwhile", time.Time{}, func(r *compositionRuns, key string) error {
_, err := r.endRun(key, "run-1", meshwants.CompositionRunOutcomePassed, "", nil)
return err
}},
} {
t.Run(c.name, func(t *testing.T) {
kv := newRunsMem()
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
key := runKeyOf("main.3", "n1")
if !c.ended.IsZero() {
_, _, _ = r.change(key, func(run *meshwants.CompositionRun) bool { run.Ended, run.Outcome = &c.ended, "failed"; return true })
}
_ = kv.arm(runBoundPrefix+key, time.Minute)
kv.beforeDel = func() { // a rerun replaces the key and arms its untaken bound
for {
raw, rev, _ := kv.get(key)
var run meshwants.CompositionRun
_ = json.Unmarshal(raw, &run)
if run.Ended == nil {
at := now
run.Ended, run.Outcome = &at, "failed"
}
fresh, _ := json.Marshal(meshwants.CompositionRun{Candidate: "main.3", Machine: "n1", Ask: "run-2", Started: &now, Readers: run.Readers})
if _, err := kv.update(key, fresh, rev); err == nil {
break
}
}
_ = kv.arm(runUntakenPrefix+key, time.Hour)
}
if err := c.act(r, key); err != nil {
t.Fatal(err)
}
if run, _, ok, _ := r.read(key); !ok || run.Ask != "run-2" || run.Ended != nil {
t.Errorf("the key holds %+v (stands %v), want the rerun's running run-2", run, ok)
}
if v, _, _ := kv.get(runUntakenPrefix + key); v == nil {
t.Error("the untaken bound the rerun's run armed is deleted")
}
})
}
t.Run("settings", func(t *testing.T) {
dir := t.TempDir()
files := map[string]string{"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "8", "MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": "2h",
@@ -89,12 +89,10 @@ func (c *compositionSteps) drop(r *compositionRuns, record func(meshhandactsclie
answer.Ended = append(answer.Ended, reader)
}
}
var errs []error
for _, k := range []string{key, runBoundPrefix + key, runUntakenPrefix + key} {
errs = append(errs, r.kv.del(k))
}
// The dropped run's key goes while it holds that run: a run started on it since stands.
err = r.deleteRun(key, func(c meshwants.CompositionRun) bool { return c.Ask == run.Ask })
r.heard()
return answer, errors.Join(errs...)
return answer, err
}
// compositionVerbs are the process's handlers of the seat mesh-wants' verbs compositions and compositions-drop.