assigner: a composition run is found by the keys its state's watch handed, beside the listing
A bucket's listing misses a key that is being written at that moment (97 of 300 listings on an embedded bus while one key was updated), so a run's end, its ask-taken or a moved reader could miss the run being written. Every lookup of composition-runs now takes the keys the state's own watch handed beside the listing. Builds: design 47 G §719 (the assigner's own watch of its state), §720, §727; A-468, A-470d
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -176,7 +177,7 @@ func (r *compositionRuns) endRun(key, ask, outcome, said string, verdict map[str
|
||||
// readersOfOpen are the readers of every run with no `ended`, each with its ask's id once: what the assigner takes up again at
|
||||
// its start (§719), each reader's end said from the keys alone (A-468).
|
||||
func (r *compositionRuns) readersOfOpen() ([]meshwants.CompositionReader, error) {
|
||||
all, err := r.kv.keys()
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -202,7 +203,7 @@ func (r *compositionRuns) readersOfOpen() ([]meshwants.CompositionReader, error)
|
||||
|
||||
// keysOf are the keys of the candidate's runs that the reader reads.
|
||||
func (r *compositionRuns) keysOf(candidate, id string) ([]string, error) {
|
||||
all, err := r.kv.keys()
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -216,6 +217,7 @@ func (r *compositionRuns) keysOf(candidate, id string) ([]string, error) {
|
||||
out = append(out, key)
|
||||
}
|
||||
}
|
||||
slices.Sort(out) // by machine, as a step names its machines
|
||||
return out, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -219,6 +219,9 @@ type compositionRuns struct {
|
||||
mu sync.Mutex
|
||||
// changed is closed and replaced at each change of the state heard (heard): a step waiting on a run reads its key again.
|
||||
changed chan struct{}
|
||||
// seen are the keys the state's watch handed and has not seen deleted: a listing of the bucket misses a key that is
|
||||
// being written at that moment, so every lookup takes these beside it (keys).
|
||||
seen map[string]bool
|
||||
// say says an event of the seat mesh-wants (a reader's wait, A-469y); drop asks the build seat's drop of a run ask (§119);
|
||||
// nil says or drops nothing.
|
||||
say publisher
|
||||
@@ -240,6 +243,36 @@ func (r *compositionRuns) heard() {
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// saw keeps a key the state's watch handed: standing, or deleted or expired.
|
||||
func (r *compositionRuns) saw(key string, standing bool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.seen == nil {
|
||||
r.seen = map[string]bool{}
|
||||
}
|
||||
if standing {
|
||||
r.seen[key] = true
|
||||
} else {
|
||||
delete(r.seen, key)
|
||||
}
|
||||
}
|
||||
|
||||
// keys are the state's keys: the bucket's listing with the keys its watch handed beside it, sorted. A key read gone later
|
||||
// is skipped by its reader.
|
||||
func (r *compositionRuns) keys() ([]string, error) {
|
||||
all, err := r.kv.keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.mu.Lock()
|
||||
for key := range r.seen {
|
||||
all = append(all, key)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
slices.Sort(all)
|
||||
return slices.Compact(all), nil
|
||||
}
|
||||
|
||||
// wake is closed at the next change heard.
|
||||
func (r *compositionRuns) wake() <-chan struct{} {
|
||||
r.mu.Lock()
|
||||
@@ -379,7 +412,7 @@ func (r *compositionRuns) room() error {
|
||||
|
||||
// sweep deletes, among the 20 oldest ended runs, those ended more than composition-runs-kept ago, with their bound keys.
|
||||
func (r *compositionRuns) sweep() error {
|
||||
all, err := r.kv.keys()
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -452,6 +485,7 @@ func watchRuns(ctx context.Context, kv jetstream.KeyValue, r *compositionRuns, e
|
||||
}
|
||||
continue
|
||||
}
|
||||
r.saw(e.Key(), e.Operation() == jetstream.KeyValuePut)
|
||||
if e.Operation() == jetstream.KeyValuePurge && !isRunKey(e.Key()) && expired != nil {
|
||||
expired(e.Key())
|
||||
}
|
||||
@@ -480,7 +514,7 @@ func (r *compositionRuns) end(key, ask, outcome, said string, verdict map[string
|
||||
|
||||
// keyOfAsk is the key of the run whose ask is ask, or none.
|
||||
func (r *compositionRuns) keyOfAsk(ask string) (string, meshwants.CompositionRun, error) {
|
||||
all, err := r.kv.keys()
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return "", meshwants.CompositionRun{}, err
|
||||
}
|
||||
@@ -586,7 +620,7 @@ func (r *compositionRuns) sharedOf(keys []string, runs map[string]meshwants.Comp
|
||||
// their asks' ids, each told to stop before a run is ended; a running run left with no reader has its ask dropped and ends could not run, "no reader is left" (A-469z),
|
||||
// so a later step of the candidate starts a new run. A run other readers read goes on for them.
|
||||
func (r *compositionRuns) leave(pipeline string, stop func(ids []string)) ([]string, error) {
|
||||
all, err := r.kv.keys()
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -211,6 +211,25 @@ func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) {
|
||||
t.Errorf("keys %v, want %v: a run ended more than a day ago goes with its bound key at the next run", keys, want)
|
||||
}
|
||||
})
|
||||
t.Run("a listing that misses a key being written", func(t *testing.T) {
|
||||
kv := &missingRuns{runsMem: newRunsMem()}
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv.hide = runKeyOf("main.3", "n1")
|
||||
if key, _, _ := r.keyOfAsk("run-1"); key != "" {
|
||||
t.Fatalf("the stand's listing does not hide the key")
|
||||
}
|
||||
r.saw(kv.hide, true) // as the state's watch hands it
|
||||
if key, _, _ := r.keyOfAsk("run-1"); key != kv.hide {
|
||||
t.Errorf("the run of ask run-1 is not found by the keys the watch handed: %q", key)
|
||||
}
|
||||
r.saw(kv.hide, false)
|
||||
if keys, _ := r.keys(); slices.Contains(keys, kv.hide) {
|
||||
t.Errorf("a key the watch saw deleted is still listed: %v", keys)
|
||||
}
|
||||
})
|
||||
t.Run("settings", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
files := map[string]string{"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "8", "MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": "2h",
|
||||
@@ -355,3 +374,14 @@ func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// missingRuns is runsMem whose listing misses one key, as a bucket's listing misses a key being written at that moment.
|
||||
type missingRuns struct {
|
||||
*runsMem
|
||||
hide string
|
||||
}
|
||||
|
||||
func (k *missingRuns) keys() ([]string, error) {
|
||||
all, err := k.runsMem.keys()
|
||||
return slices.DeleteFunc(all, func(key string) bool { return key == k.hide }), err
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ const (
|
||||
// compositions answers each composition run the state keeps, under §719's field names plus its key, with the candidate and the
|
||||
// machine named only those, and the state's fill against its cap.
|
||||
func (r *compositionRuns) compositions(in meshwants.CompositionsInput) (meshwants.CompositionsAnswer, error) {
|
||||
all, err := r.kv.keys()
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return meshwants.CompositionsAnswer{}, fmt.Errorf("the state %s is not listed: %w", "composition-runs", err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user