assigner: a composition run is found by the keys its state's watch handed, beside the listing

A bucket's listing misses a key that is being written at that moment (97 of 300 listings on
an embedded bus while one key was updated), so a run's end, its ask-taken or a moved reader
could miss the run being written. Every lookup of composition-runs now takes the keys the
state's own watch handed beside the listing.

Builds: design 47 G §719 (the assigner's own watch of its state), §720, §727; A-468, A-470d
This commit is contained in:
2026-11-17 12:07:56 +01:00
parent eadfcd4cf8
commit 930a407a3a
4 changed files with 72 additions and 6 deletions
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"slices"
"strings"
"time"
@@ -176,7 +177,7 @@ func (r *compositionRuns) endRun(key, ask, outcome, said string, verdict map[str
// readersOfOpen are the readers of every run with no `ended`, each with its ask's id once: what the assigner takes up again at
// its start (§719), each reader's end said from the keys alone (A-468).
func (r *compositionRuns) readersOfOpen() ([]meshwants.CompositionReader, error) {
all, err := r.kv.keys()
all, err := r.keys()
if err != nil {
return nil, err
}
@@ -202,7 +203,7 @@ func (r *compositionRuns) readersOfOpen() ([]meshwants.CompositionReader, error)
// keysOf are the keys of the candidate's runs that the reader reads.
func (r *compositionRuns) keysOf(candidate, id string) ([]string, error) {
all, err := r.kv.keys()
all, err := r.keys()
if err != nil {
return nil, err
}
@@ -216,6 +217,7 @@ func (r *compositionRuns) keysOf(candidate, id string) ([]string, error) {
out = append(out, key)
}
}
slices.Sort(out) // by machine, as a step names its machines
return out, nil
}
@@ -219,6 +219,9 @@ type compositionRuns struct {
mu sync.Mutex
// changed is closed and replaced at each change of the state heard (heard): a step waiting on a run reads its key again.
changed chan struct{}
// seen are the keys the state's watch handed and has not seen deleted: a listing of the bucket misses a key that is
// being written at that moment, so every lookup takes these beside it (keys).
seen map[string]bool
// say says an event of the seat mesh-wants (a reader's wait, A-469y); drop asks the build seat's drop of a run ask (§119);
// nil says or drops nothing.
say publisher
@@ -240,6 +243,36 @@ func (r *compositionRuns) heard() {
r.mu.Unlock()
}
// saw keeps a key the state's watch handed: standing, or deleted or expired.
func (r *compositionRuns) saw(key string, standing bool) {
r.mu.Lock()
defer r.mu.Unlock()
if r.seen == nil {
r.seen = map[string]bool{}
}
if standing {
r.seen[key] = true
} else {
delete(r.seen, key)
}
}
// keys are the state's keys: the bucket's listing with the keys its watch handed beside it, sorted. A key read gone later
// is skipped by its reader.
func (r *compositionRuns) keys() ([]string, error) {
all, err := r.kv.keys()
if err != nil {
return nil, err
}
r.mu.Lock()
for key := range r.seen {
all = append(all, key)
}
r.mu.Unlock()
slices.Sort(all)
return slices.Compact(all), nil
}
// wake is closed at the next change heard.
func (r *compositionRuns) wake() <-chan struct{} {
r.mu.Lock()
@@ -379,7 +412,7 @@ func (r *compositionRuns) room() error {
// sweep deletes, among the 20 oldest ended runs, those ended more than composition-runs-kept ago, with their bound keys.
func (r *compositionRuns) sweep() error {
all, err := r.kv.keys()
all, err := r.keys()
if err != nil {
return err
}
@@ -452,6 +485,7 @@ func watchRuns(ctx context.Context, kv jetstream.KeyValue, r *compositionRuns, e
}
continue
}
r.saw(e.Key(), e.Operation() == jetstream.KeyValuePut)
if e.Operation() == jetstream.KeyValuePurge && !isRunKey(e.Key()) && expired != nil {
expired(e.Key())
}
@@ -480,7 +514,7 @@ func (r *compositionRuns) end(key, ask, outcome, said string, verdict map[string
// keyOfAsk is the key of the run whose ask is ask, or none.
func (r *compositionRuns) keyOfAsk(ask string) (string, meshwants.CompositionRun, error) {
all, err := r.kv.keys()
all, err := r.keys()
if err != nil {
return "", meshwants.CompositionRun{}, err
}
@@ -586,7 +620,7 @@ func (r *compositionRuns) sharedOf(keys []string, runs map[string]meshwants.Comp
// their asks' ids, each told to stop before a run is ended; a running run left with no reader has its ask dropped and ends could not run, "no reader is left" (A-469z),
// so a later step of the candidate starts a new run. A run other readers read goes on for them.
func (r *compositionRuns) leave(pipeline string, stop func(ids []string)) ([]string, error) {
all, err := r.kv.keys()
all, err := r.keys()
if err != nil {
return nil, err
}
@@ -211,6 +211,25 @@ func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) {
t.Errorf("keys %v, want %v: a run ended more than a day ago goes with its bound key at the next run", keys, want)
}
})
t.Run("a listing that misses a key being written", func(t *testing.T) {
kv := &missingRuns{runsMem: newRunsMem()}
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
kv.hide = runKeyOf("main.3", "n1")
if key, _, _ := r.keyOfAsk("run-1"); key != "" {
t.Fatalf("the stand's listing does not hide the key")
}
r.saw(kv.hide, true) // as the state's watch hands it
if key, _, _ := r.keyOfAsk("run-1"); key != kv.hide {
t.Errorf("the run of ask run-1 is not found by the keys the watch handed: %q", key)
}
r.saw(kv.hide, false)
if keys, _ := r.keys(); slices.Contains(keys, kv.hide) {
t.Errorf("a key the watch saw deleted is still listed: %v", keys)
}
})
t.Run("settings", func(t *testing.T) {
dir := t.TempDir()
files := map[string]string{"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "8", "MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": "2h",
@@ -355,3 +374,14 @@ func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) {
}
})
}
// missingRuns is runsMem whose listing misses one key, as a bucket's listing misses a key being written at that moment.
type missingRuns struct {
*runsMem
hide string
}
func (k *missingRuns) keys() ([]string, error) {
all, err := k.runsMem.keys()
return slices.DeleteFunc(all, func(key string) bool { return key == k.hide }), err
}
@@ -26,7 +26,7 @@ const (
// compositions answers each composition run the state keeps, under §719's field names plus its key, with the candidate and the
// machine named only those, and the state's fill against its cap.
func (r *compositionRuns) compositions(in meshwants.CompositionsInput) (meshwants.CompositionsAnswer, error) {
all, err := r.kv.keys()
all, err := r.keys()
if err != nil {
return meshwants.CompositionsAnswer{}, fmt.Errorf("the state %s is not listed: %w", "composition-runs", err)
}