From 930a407a3ad6f1eb709c93d4cba2da814a36cc65 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 12:07:56 +0100 Subject: [PATCH] assigner: a composition run is found by the keys its state's watch handed, beside the listing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A bucket's listing misses a key that is being written at that moment (97 of 300 listings on an embedded bus while one key was updated), so a run's end, its ask-taken or a moved reader could miss the run being written. Every lookup of composition-runs now takes the keys the state's own watch handed beside the listing. Builds: design 47 G §719 (the assigner's own watch of its state), §720, §727; A-468, A-470d --- .../compositionbounds.go | 6 ++- .../mesh-delivery-assigner/compositionruns.go | 40 +++++++++++++++++-- .../compositionruns_test.go | 30 ++++++++++++++ .../compositions_tools.go | 2 +- 4 files changed, 72 insertions(+), 6 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go index d1b83bf0..a872cce4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionbounds.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "slices" "strings" "time" @@ -176,7 +177,7 @@ func (r *compositionRuns) endRun(key, ask, outcome, said string, verdict map[str // readersOfOpen are the readers of every run with no `ended`, each with its ask's id once: what the assigner takes up again at // its start (§719), each reader's end said from the keys alone (A-468). func (r *compositionRuns) readersOfOpen() ([]meshwants.CompositionReader, error) { - all, err := r.kv.keys() + all, err := r.keys() if err != nil { return nil, err } @@ -202,7 +203,7 @@ func (r *compositionRuns) readersOfOpen() ([]meshwants.CompositionReader, error) // keysOf are the keys of the candidate's runs that the reader reads. func (r *compositionRuns) keysOf(candidate, id string) ([]string, error) { - all, err := r.kv.keys() + all, err := r.keys() if err != nil { return nil, err } @@ -216,6 +217,7 @@ func (r *compositionRuns) keysOf(candidate, id string) ([]string, error) { out = append(out, key) } } + slices.Sort(out) // by machine, as a step names its machines return out, nil } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go index 58015a2e..d1c3c192 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go @@ -219,6 +219,9 @@ type compositionRuns struct { mu sync.Mutex // changed is closed and replaced at each change of the state heard (heard): a step waiting on a run reads its key again. changed chan struct{} + // seen are the keys the state's watch handed and has not seen deleted: a listing of the bucket misses a key that is + // being written at that moment, so every lookup takes these beside it (keys). + seen map[string]bool // say says an event of the seat mesh-wants (a reader's wait, A-469y); drop asks the build seat's drop of a run ask (§119); // nil says or drops nothing. say publisher @@ -240,6 +243,36 @@ func (r *compositionRuns) heard() { r.mu.Unlock() } +// saw keeps a key the state's watch handed: standing, or deleted or expired. +func (r *compositionRuns) saw(key string, standing bool) { + r.mu.Lock() + defer r.mu.Unlock() + if r.seen == nil { + r.seen = map[string]bool{} + } + if standing { + r.seen[key] = true + } else { + delete(r.seen, key) + } +} + +// keys are the state's keys: the bucket's listing with the keys its watch handed beside it, sorted. A key read gone later +// is skipped by its reader. +func (r *compositionRuns) keys() ([]string, error) { + all, err := r.kv.keys() + if err != nil { + return nil, err + } + r.mu.Lock() + for key := range r.seen { + all = append(all, key) + } + r.mu.Unlock() + slices.Sort(all) + return slices.Compact(all), nil +} + // wake is closed at the next change heard. func (r *compositionRuns) wake() <-chan struct{} { r.mu.Lock() @@ -379,7 +412,7 @@ func (r *compositionRuns) room() error { // sweep deletes, among the 20 oldest ended runs, those ended more than composition-runs-kept ago, with their bound keys. func (r *compositionRuns) sweep() error { - all, err := r.kv.keys() + all, err := r.keys() if err != nil { return err } @@ -452,6 +485,7 @@ func watchRuns(ctx context.Context, kv jetstream.KeyValue, r *compositionRuns, e } continue } + r.saw(e.Key(), e.Operation() == jetstream.KeyValuePut) if e.Operation() == jetstream.KeyValuePurge && !isRunKey(e.Key()) && expired != nil { expired(e.Key()) } @@ -480,7 +514,7 @@ func (r *compositionRuns) end(key, ask, outcome, said string, verdict map[string // keyOfAsk is the key of the run whose ask is ask, or none. func (r *compositionRuns) keyOfAsk(ask string) (string, meshwants.CompositionRun, error) { - all, err := r.kv.keys() + all, err := r.keys() if err != nil { return "", meshwants.CompositionRun{}, err } @@ -586,7 +620,7 @@ func (r *compositionRuns) sharedOf(keys []string, runs map[string]meshwants.Comp // their asks' ids, each told to stop before a run is ended; a running run left with no reader has its ask dropped and ends could not run, "no reader is left" (A-469z), // so a later step of the candidate starts a new run. A run other readers read goes on for them. func (r *compositionRuns) leave(pipeline string, stop func(ids []string)) ([]string, error) { - all, err := r.kv.keys() + all, err := r.keys() if err != nil { return nil, err } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go index a7f73491..0780befe 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go @@ -211,6 +211,25 @@ func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) { t.Errorf("keys %v, want %v: a run ended more than a day ago goes with its bound key at the next run", keys, want) } }) + t.Run("a listing that misses a key being written", func(t *testing.T) { + kv := &missingRuns{runsMem: newRunsMem()} + r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf) + if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil { + t.Fatal(err) + } + kv.hide = runKeyOf("main.3", "n1") + if key, _, _ := r.keyOfAsk("run-1"); key != "" { + t.Fatalf("the stand's listing does not hide the key") + } + r.saw(kv.hide, true) // as the state's watch hands it + if key, _, _ := r.keyOfAsk("run-1"); key != kv.hide { + t.Errorf("the run of ask run-1 is not found by the keys the watch handed: %q", key) + } + r.saw(kv.hide, false) + if keys, _ := r.keys(); slices.Contains(keys, kv.hide) { + t.Errorf("a key the watch saw deleted is still listed: %v", keys) + } + }) t.Run("settings", func(t *testing.T) { dir := t.TempDir() files := map[string]string{"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "8", "MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": "2h", @@ -355,3 +374,14 @@ func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) { } }) } + +// missingRuns is runsMem whose listing misses one key, as a bucket's listing misses a key being written at that moment. +type missingRuns struct { + *runsMem + hide string +} + +func (k *missingRuns) keys() ([]string, error) { + all, err := k.runsMem.keys() + return slices.DeleteFunc(all, func(key string) bool { return key == k.hide }), err +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositions_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositions_tools.go index 25fc77a4..e87444de 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositions_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositions_tools.go @@ -26,7 +26,7 @@ const ( // compositions answers each composition run the state keeps, under §719's field names plus its key, with the candidate and the // machine named only those, and the state's fill against its cap. func (r *compositionRuns) compositions(in meshwants.CompositionsInput) (meshwants.CompositionsAnswer, error) { - all, err := r.kv.keys() + all, err := r.keys() if err != nil { return meshwants.CompositionsAnswer{}, fmt.Errorf("the state %s is not listed: %w", "composition-runs", err) }