diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go index a872cce4..c3776096 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go @@ -88,7 +88,7 @@ func (r *compositionRuns) heardAskTaken(e nodebuildgo.AskTaken) error { return fmt.Errorf("the bound of the composition run %s is not armed: %w", key, err) } if r.untaken(key) { - if err := r.kv.del(runUntakenPrefix + key); err != nil { + if err := r.clearBounds(key, e.Ask, runUntakenPrefix); err != nil { return err } r.sayWaits(run, run.Readers, "ran") @@ -166,10 +166,8 @@ func (r *compositionRuns) endRun(key, ask, outcome, said string, verdict map[str r.sayWaits(run, run.Readers, "superseded") } } - for _, k := range []string{runBoundPrefix + key, runUntakenPrefix + key} { - if err := r.kv.del(k); err != nil { - r.logf("the bound key %s of the ended composition run is not deleted: %v", k, err) - } + if err := r.clearBounds(key, ask, runBoundPrefix, runUntakenPrefix); err != nil { + r.logf("the bound keys of the ended composition run %s are not deleted: %v", key, err) } return true, nil } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go index d1c3c192..63005ea7 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go @@ -130,7 +130,9 @@ type runsKV interface { // create writes a key that does not stand, update one at its revision: errRunChanged when it does not stand so. create(key string, value []byte) (uint64, error) update(key string, value []byte, revision uint64) (uint64, error) - del(key string) error + // delAt deletes a key at its revision: errRunChanged when it no longer stands so, nothing for a key not there. No write + // of the state is made from a value read before its latest change. + delAt(key string, revision uint64) error keys() ([]string, error) // arm writes a bound key afresh with its own time to live (ยง720): its expiry is the bus's event. arm(key string, ttl time.Duration) error @@ -178,6 +180,14 @@ func (b busRuns) del(key string) error { return nil } +func (b busRuns) delAt(key string, revision uint64) error { + err := b.kv.Delete(b.ctx, key, jetstream.LastRevision(revision)) + if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + return errRunChanged + } + return err +} + func (b busRuns) keys() ([]string, error) { l, err := b.kv.ListKeys(b.ctx) if err != nil { @@ -438,7 +448,60 @@ func (r *compositionRuns) sweep() error { var errs []error for _, e := range ended { if r.now().Sub(e.at) >= r.set.kept { - errs = append(errs, r.kv.del(e.key), r.kv.del(runBoundPrefix+e.key), r.kv.del(runUntakenPrefix+e.key)) + errs = append(errs, r.deleteRun(e.key, func(run meshwants.CompositionRun) bool { + return run.Ended != nil && r.now().Sub(*run.Ended) >= r.set.kept + })) + } + } + return errors.Join(errs...) +} + +// deleteRun deletes a run's key with its bound keys while the value it holds is still one gone judges gone, each delete at the +// revision just read and judged again on a conflict: a run that replaced the key meanwhile, and its bounds, stand. +func (r *compositionRuns) deleteRun(key string, gone func(meshwants.CompositionRun) bool) error { + for { + run, rev, ok, err := r.read(key) + if err != nil || !ok || !gone(run) { + return err + } + if err := r.clearBounds(key, run.Ask, runBoundPrefix, runUntakenPrefix); err != nil { + return err + } + if err := r.kv.delAt(key, rev); errors.Is(err, errRunChanged) { + continue + } else if err != nil { + return err + } + r.heard() + return nil + } +} + +// clearBounds deletes the bound keys of the run of ask while key holds that run (or none): each at the revision it was read +// at, read again on a conflict, so a bound a newer run of the key armed is never deleted. +func (r *compositionRuns) clearBounds(key, ask string, prefixes ...string) error { + var errs []error + for _, prefix := range prefixes { + for { + run, _, ok, err := r.read(key) + if err != nil { + errs = append(errs, err) + break + } + if ok && run.Ask != ask { + break // a newer run holds the key: its bounds are its own + } + raw, rev, err := r.kv.get(prefix + key) + if err != nil || raw == nil { + errs = append(errs, err) + break + } + if err := r.kv.delAt(prefix+key, rev); errors.Is(err, errRunChanged) { + continue + } else if err != nil { + errs = append(errs, err) + } + break } } return errors.Join(errs...) @@ -735,14 +798,14 @@ func (r *compositionRuns) rejoin(reader meshwants.CompositionReader, candidate, fresh.Readers = append(fresh.Readers, c) } fresh.Readers = append(fresh.Readers, reader) + if err := r.clearBounds(key, run.Ask, runBoundPrefix, runUntakenPrefix); err != nil { // the former run's bounds go with it + r.logf("the bound keys of the former composition run %s are not deleted: %v", key, err) + } if err := r.write(key, fresh, rev); errors.Is(err, errRunChanged) { continue } else if err != nil { return joined{}, err } - for _, k := range []string{runBoundPrefix + key, runUntakenPrefix + key} { - _ = r.kv.del(k) // the former run's bounds go with it - } if err := r.kv.arm(runUntakenPrefix+key, r.set.untaken); err != nil { r.logf("the untaken bound of the composition run %s is not armed: %v", key, err) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go index 0780befe..4bbd6d37 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go @@ -24,6 +24,18 @@ type runsMem struct { ttl map[string]time.Duration bytes int64 n uint64 + // beforeDel, when set, runs once before the next delete: another write landing between a read and the delete. + beforeDel func() +} + +func (k *runsMem) another() { + k.mu.Lock() + f := k.beforeDel + k.beforeDel = nil + k.mu.Unlock() + if f != nil { + f() + } } func newRunsMem() *runsMem { @@ -58,9 +70,16 @@ func (k *runsMem) update(key string, v []byte, rev uint64) (uint64, error) { return k.n, nil } -func (k *runsMem) del(key string) error { +func (k *runsMem) delAt(key string, rev uint64) error { + k.another() k.mu.Lock() defer k.mu.Unlock() + if _, ok := k.vals[key]; !ok { + return nil + } + if k.revs[key] != rev { + return errRunChanged + } delete(k.vals, key) delete(k.revs, key) delete(k.ttl, key) @@ -230,6 +249,57 @@ func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) { t.Errorf("a key the watch saw deleted is still listed: %v", keys) } }) + // No delete lands on a value newer than the one it was judged from (the kind test 109 met in test-runs): a rerun's run + // replacing the key, and its untaken bound, stand. + for _, c := range []struct { + name string + ended time.Time + act func(r *compositionRuns, key string) error + }{ + {"the sweep of a run replaced meanwhile", now.Add(-25 * time.Hour), func(r *compositionRuns, _ string) error { return r.sweep() }}, + {"the end of a run replaced meanwhile", time.Time{}, func(r *compositionRuns, key string) error { + _, err := r.endRun(key, "run-1", meshwants.CompositionRunOutcomePassed, "", nil) + return err + }}, + } { + t.Run(c.name, func(t *testing.T) { + kv := newRunsMem() + r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf) + if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil { + t.Fatal(err) + } + key := runKeyOf("main.3", "n1") + if !c.ended.IsZero() { + _, _, _ = r.change(key, func(run *meshwants.CompositionRun) bool { run.Ended, run.Outcome = &c.ended, "failed"; return true }) + } + _ = kv.arm(runBoundPrefix+key, time.Minute) + kv.beforeDel = func() { // a rerun replaces the key and arms its untaken bound + for { + raw, rev, _ := kv.get(key) + var run meshwants.CompositionRun + _ = json.Unmarshal(raw, &run) + if run.Ended == nil { + at := now + run.Ended, run.Outcome = &at, "failed" + } + fresh, _ := json.Marshal(meshwants.CompositionRun{Candidate: "main.3", Machine: "n1", Ask: "run-2", Started: &now, Readers: run.Readers}) + if _, err := kv.update(key, fresh, rev); err == nil { + break + } + } + _ = kv.arm(runUntakenPrefix+key, time.Hour) + } + if err := c.act(r, key); err != nil { + t.Fatal(err) + } + if run, _, ok, _ := r.read(key); !ok || run.Ask != "run-2" || run.Ended != nil { + t.Errorf("the key holds %+v (stands %v), want the rerun's running run-2", run, ok) + } + if v, _, _ := kv.get(runUntakenPrefix + key); v == nil { + t.Error("the untaken bound the rerun's run armed is deleted") + } + }) + } t.Run("settings", func(t *testing.T) { dir := t.TempDir() files := map[string]string{"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "8", "MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": "2h", diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositions_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositions_tools.go index e87444de..29ce684b 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositions_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositions_tools.go @@ -89,12 +89,10 @@ func (c *compositionSteps) drop(r *compositionRuns, record func(meshhandactsclie answer.Ended = append(answer.Ended, reader) } } - var errs []error - for _, k := range []string{key, runBoundPrefix + key, runUntakenPrefix + key} { - errs = append(errs, r.kv.del(k)) - } + // The dropped run's key goes while it holds that run: a run started on it since stands. + err = r.deleteRun(key, func(c meshwants.CompositionRun) bool { return c.Ask == run.Ask }) r.heard() - return answer, errors.Join(errs...) + return answer, err } // compositionVerbs are the process's handlers of the seat mesh-wants' verbs compositions and compositions-drop.