From ef5e5d13e07b8868122bbcc2cf63d59b045e4cf9 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 11:26:51 +0100 Subject: [PATCH] assigner: the state composition-runs, one key per candidate and machine (A1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A composition run is kept once per candidate and machine, so steps of one candidate share it: the key is written create-only, a reader is the step's StepRun with its ask id, a run that could not run is replaced, a run past composition-runs-kept goes at the next start, and at the cap no run starts, naming the fill. The settings reach the process through resource files. Builds: design 47 G §718, §719, §720 (could not run is never read); A-468, A-468b, A-468c, A-469d, A-469w, A-469x, A-470 --- .../mesh-delivery-assigner/compositionrun.go | 2 + .../mesh-delivery-assigner/compositionruns.go | 443 ++++++++++++++++++ .../compositionruns_test.go | 229 +++++++++ .../integration_test.go | 2 +- .../cmd/mesh-delivery-assigner/main.go | 25 +- .../internal/buckets/buckets.go | 24 +- .../internal/buckets/buckets_test.go | 3 +- .../mesh-delivery-assigner/manifest_test.go | 12 +- modules/mesh-delivery-assigner/module.json | 82 +++- 9 files changed, 799 insertions(+), 23 deletions(-) create mode 100644 modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go create mode 100644 modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go index 3422319c..b14d3ce7 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go @@ -109,6 +109,8 @@ func (a *awayNodes) is(machine string) bool { // compositionRun runs a composition step. type compositionRun struct { + // runs are the composition runs of the state composition-runs (compositionruns.go). + runs *compositionRuns inputs *compositionInputs keeper *inputKeeper // call asks a seat verb held once for the mesh (process.Conn.CallKept). diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go new file mode 100644 index 00000000..c235ed5a --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns.go @@ -0,0 +1,443 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "slices" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-sdk/go/keys" + + "mesh-delivery-assigner/internal/buckets" +) + +// The composition runs (design 47 G §718, §719; A-468, A-469d, A-469w, A-469x): the assigner composes each machine once per +// candidate, whatever number of composition steps name it. Each run is one key of its old state `composition-runs`, +// `.`, each part escaped as a key's name; the step that starts a run writes the key create-only, +// so two steps of one candidate starting at once start one run and the second reads it. A reader is the composition step's +// StepRun with its ask's id (A-468), added and removed by compare-and-set. A key is deleted once composition-runs-kept has +// passed since its `ended`, judged at each new run over at most 20 of the oldest ended keys, as the state inputs is +// (A-465b, A-468b); at the cap (composition-runs-cap-mib, the manifest's max-mib its ceiling, A-469d) no run is started. + +const ( + // runsSwept is how many of the oldest ended keys each new run judges. + runsSwept = 20 + // runBoundPrefix and runUntakenPrefix open a run's two bound keys, each with its own time to live (§720, A-468b, + // A-470): `bound.` from the ask's ask-taken, `untaken.` from the run's start. + runBoundPrefix = "bound." + runUntakenPrefix = "untaken." +) + +// runKeyOf is the key of a candidate's composition run of a machine (§719). +func runKeyOf(candidate, machine string) string { + return keys.EscapeName(candidate) + "." + keys.EscapeName(machine) +} + +// isRunKey is whether a key of the state is a run's own, not one of its bound keys. +func isRunKey(key string) bool { + return !strings.HasPrefix(key, runBoundPrefix) && !strings.HasPrefix(key, runUntakenPrefix) +} + +// runSettings are the settings of the composition runs, each read at the start from the file the manifest declares. +type runSettings struct { + // capBytes is composition-runs-cap-mib in bytes. + capBytes int64 + // kept is composition-runs-kept, stepBound composition-step-bound, untaken run-untaken-bound (A-469x). + kept, stepBound, untaken time.Duration + // blameTries is blame-tries, carried in each run's request (§721). + blameTries int +} + +// defaultRunSettings are the settings' defaults, as the manifest gives them. +var defaultRunSettings = runSettings{capBytes: 16 << 20, kept: 24 * time.Hour, stepBound: 30 * time.Minute, untaken: 24 * time.Hour, blameTries: 20} + +// runSettingsOf reads the settings composition-runs-cap-mib, composition-runs-kept, composition-step-bound, +// run-untaken-bound and blame-tries from the files the environment names (A-469x, as outsideSettings): a missing or empty +// file is its default; a content that is no positive whole number or Go duration is an error naming the file. +func runSettingsOf(getenv func(string) string) (runSettings, error) { + s := defaultRunSettings + mib, err := settingFile(getenv("MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE")) + if err != nil { + return s, err + } + if mib > 0 { + s.capBytes = int64(mib) << 20 + } + tries, err := settingFile(getenv("MESH_ASSIGNER_BLAME_TRIES_FILE")) + if err != nil { + return s, err + } + if tries > 0 { + s.blameTries = tries + } + for env, to := range map[string]*time.Duration{"MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": &s.kept, + "MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": &s.stepBound, "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": &s.untaken} { + d, err := durationFile(getenv(env)) + if err != nil { + return s, err + } + if d > 0 { + *to = d + } + } + return s, nil +} + +// durationFile reads one setting's file holding a Go duration: 0 for no path, no file or an empty one. +func durationFile(path string) (time.Duration, error) { + if path == "" { + return 0, nil + } + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return 0, nil + } + if err != nil { + return 0, err + } + v := strings.TrimSpace(string(raw)) + if v == "" { + return 0, nil + } + d, err := time.ParseDuration(v) + if err != nil || d <= 0 { + return 0, fmt.Errorf("%s holds %q, which is not a positive Go duration such as 30m", path, v) + } + return d, nil +} + +// errRunChanged is a compare-and-set of a run's key that met another write. +var errRunChanged = errors.New("the composition run's key changed") + +// runsKV is the state composition-runs: the bus bucket, or a test's. +type runsKV interface { + // get is a key's value and revision; nil for a key not there. + get(key string) ([]byte, uint64, error) + // create writes a key that does not stand, update one at its revision: errRunChanged when it does not stand so. + create(key string, value []byte) (uint64, error) + update(key string, value []byte, revision uint64) (uint64, error) + del(key string) error + keys() ([]string, error) + // arm writes a bound key afresh with its own time to live (§720): its expiry is the bus's event. + arm(key string, ttl time.Duration) error + // used is how many bytes the state holds. + used() (int64, error) +} + +// busRuns is runsKV over the bus bucket. +type busRuns struct { + ctx context.Context + kv jetstream.KeyValue +} + +func (b busRuns) get(key string) ([]byte, uint64, error) { + e, err := b.kv.Get(b.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, 0, nil + } + if err != nil { + return nil, 0, err + } + return e.Value(), e.Revision(), nil +} + +func (b busRuns) create(key string, value []byte) (uint64, error) { + rev, err := b.kv.Create(b.ctx, key, value) + if errors.Is(err, jetstream.ErrKeyExists) { + return 0, errRunChanged + } + return rev, err +} + +func (b busRuns) update(key string, value []byte, revision uint64) (uint64, error) { + rev, err := b.kv.Update(b.ctx, key, value, revision) + if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + return 0, errRunChanged + } + return rev, err +} + +func (b busRuns) del(key string) error { + if err := b.kv.Delete(b.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err + } + return nil +} + +func (b busRuns) keys() ([]string, error) { + l, err := b.kv.ListKeys(b.ctx) + if err != nil { + return nil, err + } + var out []string + for k := range l.Keys() { + out = append(out, k) + } + return out, nil +} + +// arm deletes the key (a delete, never the expiry's purge) and creates it again with its time to live. +func (b busRuns) arm(key string, ttl time.Duration) error { + if err := b.del(key); err != nil { + return err + } + _, err := b.kv.Create(b.ctx, key, []byte(`{}`), jetstream.KeyTTL(ttl)) + if errors.Is(err, jetstream.ErrKeyExists) { + return nil // armed at once by another write: its clock begins now too + } + return err +} + +func (b busRuns) used() (int64, error) { + st, err := b.kv.Status(b.ctx) + if err != nil { + return 0, err + } + return int64(st.Bytes()), nil +} + +// compositionRuns keeps the composition runs in the state composition-runs. +type compositionRuns struct { + kv runsKV + set runSettings + now func() time.Time + logf func(string, ...any) + mu sync.Mutex + // changed is closed and replaced at each change of the state heard (heard): a step waiting on a run reads its key again. + changed chan struct{} +} + +func newCompositionRuns(kv runsKV, set runSettings, now func() time.Time, logf func(string, ...any)) *compositionRuns { + return &compositionRuns{kv: kv, set: set, now: now, logf: logf, changed: make(chan struct{})} +} + +// heard takes a change of the state from its watch (a module watches its own state, principle 1). +func (r *compositionRuns) heard() { + r.mu.Lock() + close(r.changed) + r.changed = make(chan struct{}) + r.mu.Unlock() +} + +// wake is closed at the next change heard. +func (r *compositionRuns) wake() <-chan struct{} { + r.mu.Lock() + defer r.mu.Unlock() + return r.changed +} + +// read is a run's value and revision, or false. +func (r *compositionRuns) read(key string) (meshwants.CompositionRun, uint64, bool, error) { + raw, rev, err := r.kv.get(key) + if err != nil || raw == nil { + return meshwants.CompositionRun{}, 0, false, err + } + var run meshwants.CompositionRun + if err := json.Unmarshal(raw, &run); err != nil { + return meshwants.CompositionRun{}, 0, false, fmt.Errorf("the composition run %s does not read: %w", key, err) + } + return run, rev, true, nil +} + +// change writes a run's key by compare-and-set: f changes the value read and says whether to write it; false leaves it. +// It answers the value written or left, and false for a key not there. +func (r *compositionRuns) change(key string, f func(*meshwants.CompositionRun) bool) (meshwants.CompositionRun, bool, error) { + for { + run, rev, ok, err := r.read(key) + if err != nil || !ok { + return run, ok, err + } + if !f(&run) { + return run, true, nil + } + raw, err := json.Marshal(run) + if err != nil { + return run, true, err + } + if _, err := r.kv.update(key, raw, rev); errors.Is(err, errRunChanged) { + continue + } else if err != nil { + return run, true, err + } + r.heard() + return run, true, nil + } +} + +// fill is the state's fill against its cap. +func (r *compositionRuns) fill() (meshwants.Fill, error) { + at := r.now().UTC() + used, err := r.kv.used() + return meshwants.Fill{Store: buckets.CompositionRunsState, Bytes: used, Cap: r.set.capBytes, At: &at}, err +} + +// joined is a reader's place in a run: the run's key, whether this reader started it (so it composes the machine), and the +// value as it stood. +type joined struct { + key string + started bool + run meshwants.CompositionRun +} + +// join finds or starts the candidate's run of the machine for reader (§718, §719, §720): a run that stands and has not +// ended, or that ended other than could not run, is read, the reader added; none, or one that could not run, is started: +// written create-only (or replacing the ended key by compare-and-set), the reader its first, ask its run ask's id. At the +// cap no run is started, an error naming the fill. +func (r *compositionRuns) join(reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) { + key := runKeyOf(candidate, machine) + for { + run, rev, ok, err := r.read(key) + if err != nil { + return joined{}, err + } + if ok && (run.Ended == nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun) { + if hasReader(run.Readers, reader.ID) { + return joined{key: key, run: run}, nil + } + run.Readers = append(run.Readers, reader) + raw, err := json.Marshal(run) + if err != nil { + return joined{}, err + } + if _, err := r.kv.update(key, raw, rev); errors.Is(err, errRunChanged) { + continue + } else if err != nil { + return joined{}, err + } + r.heard() + return joined{key: key, run: run}, nil + } + if err := r.room(); err != nil { + return joined{}, err + } + started := r.now().UTC() + fresh := meshwants.CompositionRun{Candidate: candidate, Machine: machine, Ask: ask, Started: &started, + Readers: []meshwants.CompositionReader{reader}} + raw, err := json.Marshal(fresh) + if err != nil { + return joined{}, err + } + if ok { + _, err = r.kv.update(key, raw, rev) // could not run is never read: a new run replaces it (§720) + } else { + _, err = r.kv.create(key, raw) + } + if errors.Is(err, errRunChanged) { + continue + } else if err != nil { + return joined{}, err + } + r.heard() + return joined{key: key, started: true, run: fresh}, nil + } +} + +// room sweeps the state and says whether a new run may start: an error naming the fill at the cap (as A-465a). +func (r *compositionRuns) room() error { + if err := r.sweep(); err != nil { + r.logf("composition-runs: the keys past composition-runs-kept are not all deleted: %v", err) + } + f, err := r.fill() + if err != nil { + return fmt.Errorf("the fill of the state %s is not read, so no composition run is started: %w", buckets.CompositionRunsState, err) + } + if f.Bytes >= f.Cap { + return fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so no composition run is started", buckets.CompositionRunsState, + f.Cap>>20, f.Bytes) + } + return nil +} + +// sweep deletes, among the 20 oldest ended runs, those ended more than composition-runs-kept ago, with their bound keys. +func (r *compositionRuns) sweep() error { + all, err := r.kv.keys() + if err != nil { + return err + } + type endedAt struct { + key string + at time.Time + } + var ended []endedAt + for _, key := range all { + if !isRunKey(key) { + continue + } + run, _, ok, err := r.read(key) + if err != nil || !ok || run.Ended == nil { + continue + } + ended = append(ended, endedAt{key, *run.Ended}) + } + slices.SortFunc(ended, func(a, b endedAt) int { return a.at.Compare(b.at) }) + if len(ended) > runsSwept { + ended = ended[:runsSwept] + } + var errs []error + for _, e := range ended { + if r.now().Sub(e.at) >= r.set.kept { + errs = append(errs, r.kv.del(e.key), r.kv.del(runBoundPrefix+e.key), r.kv.del(runUntakenPrefix+e.key)) + } + } + return errors.Join(errs...) +} + +// hasReader is whether readers holds the step asked under id. +func hasReader(readers []meshwants.CompositionReader, id string) bool { + return slices.ContainsFunc(readers, func(c meshwants.CompositionReader) bool { return c.ID == id }) +} + +// readerOf is a composition step as a run's reader: its StepRun with its ask's id (A-468). +func readerOf(id string, s meshwants.StepRun) meshwants.CompositionReader { + return meshwants.CompositionReader{ID: id, Pipeline: s.Pipeline, Step: s.Step, Module: s.Module, Branch: s.Branch, Commit: s.Commit, + Revision: s.Revision, Source: s.Source, Fingerprint: s.Fingerprint, Run: s.Run, Rerun: s.Rerun, By: s.By} +} + +// stepRunOf is a reader's StepRun again, for its step-ended. +func stepRunOf(c meshwants.CompositionReader) meshwants.StepRun { + return meshwants.StepRun{Pipeline: c.Pipeline, Step: c.Step, Module: c.Module, Branch: c.Branch, Commit: c.Commit, Revision: c.Revision, + Source: c.Source, Fingerprint: c.Fingerprint, Run: c.Run, Rerun: c.Rerun, By: c.By} +} + +// watchRuns hears the changes of the state composition-runs (principle 1): each change wakes the steps waiting on a run, and +// the expiry of a bound key, the bus's marker, is handed to expired (§720). After the values that stand, ready is called. +func watchRuns(ctx context.Context, kv jetstream.KeyValue, r *compositionRuns, expired func(key string), ready func()) error { + w, err := kv.WatchAll(ctx) + if err != nil { + return err + } + go func() { + defer func() { _ = w.Stop() }() + for { + select { + case <-ctx.Done(): + return + case e, ok := <-w.Updates(): + if !ok { + return + } + if e == nil { + if ready != nil { + ready() + ready = nil + } + continue + } + if e.Operation() == jetstream.KeyValuePurge && !isRunKey(e.Key()) && expired != nil { + expired(e.Key()) + } + r.heard() + } + } + }() + return nil +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go new file mode 100644 index 00000000..03171fd6 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_test.go @@ -0,0 +1,229 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "time" + + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" +) + +// runsMem is the state composition-runs in memory, its bound keys' times to live kept beside. +type runsMem struct { + mu sync.Mutex + vals map[string][]byte + revs map[string]uint64 + ttl map[string]time.Duration + bytes int64 + n uint64 +} + +func newRunsMem() *runsMem { + return &runsMem{vals: map[string][]byte{}, revs: map[string]uint64{}, ttl: map[string]time.Duration{}} +} + +func (k *runsMem) get(key string) ([]byte, uint64, error) { + k.mu.Lock() + defer k.mu.Unlock() + return k.vals[key], k.revs[key], nil +} + +func (k *runsMem) create(key string, v []byte) (uint64, error) { + k.mu.Lock() + defer k.mu.Unlock() + if _, ok := k.vals[key]; ok { + return 0, errRunChanged + } + k.n++ + k.vals[key], k.revs[key] = v, k.n + return k.n, nil +} + +func (k *runsMem) update(key string, v []byte, rev uint64) (uint64, error) { + k.mu.Lock() + defer k.mu.Unlock() + if k.revs[key] != rev { + return 0, errRunChanged + } + k.n++ + k.vals[key], k.revs[key] = v, k.n + return k.n, nil +} + +func (k *runsMem) del(key string) error { + k.mu.Lock() + defer k.mu.Unlock() + delete(k.vals, key) + delete(k.revs, key) + delete(k.ttl, key) + return nil +} + +func (k *runsMem) keys() ([]string, error) { + k.mu.Lock() + defer k.mu.Unlock() + var out []string + for key := range k.vals { + out = append(out, key) + } + slices.Sort(out) + return out, nil +} + +func (k *runsMem) arm(key string, ttl time.Duration) error { + k.mu.Lock() + defer k.mu.Unlock() + k.n++ + k.vals[key], k.revs[key], k.ttl[key] = []byte(`{}`), k.n, ttl + return nil +} + +func (k *runsMem) used() (int64, error) { + k.mu.Lock() + defer k.mu.Unlock() + if k.bytes > 0 { + return k.bytes, nil + } + var n int64 + for _, v := range k.vals { + n += int64(len(v)) + } + return n, nil +} + +// Design 47 G §718, §719, §720, A-468, A-469d, A-469x: a run's key is `.`, each part escaped; two +// steps of one candidate joining at once start one run and the second reads it; a run that could not run is replaced; at the +// cap no run starts, naming the fill; a run is deleted composition-runs-kept after its end; the settings are read from their files. +func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) { + now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC) + step := func(id, module string) meshwants.CompositionReader { + return readerOf(id, meshwants.StepRun{Pipeline: "main.3." + module, Step: meshwants.StepRunStepComposition, Module: module, Branch: "main", Revision: 3}) + } + t.Run("key", func(t *testing.T) { + if k := runKeyOf("main.3", "n1"); k != "main=2e3.n1" { + t.Errorf("the key is %q, want each part escaped as a key's name", k) + } + if isRunKey(runBoundPrefix+"main=2e3.n1") || isRunKey(runUntakenPrefix+"main=2e3.n1") || !isRunKey("main=2e3.n1") { + t.Error("a bound key is taken for a run's or a run's for a bound key") + } + }) + t.Run("one run for two steps at once", func(t *testing.T) { + kv := newRunsMem() + r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf) + var wg sync.WaitGroup + got := make([]joined, 2) + for i, m := range []string{"a", "b"} { + wg.Add(1) + go func() { + defer wg.Done() + j, err := r.join(step("ask-"+m, m), "main.3", "n1", "run-"+m) + if err != nil { + t.Error(err) + } + got[i] = j + }() + } + wg.Wait() + if got[0].started == got[1].started { + t.Fatalf("started %v and %v: want exactly one step to start the run", got[0].started, got[1].started) + } + keys, _ := kv.keys() + run, _, _, _ := r.read(runKeyOf("main.3", "n1")) + if len(keys) != 1 || len(run.Readers) != 2 || run.Candidate != "main.3" || run.Machine != "n1" || run.Started == nil { + t.Errorf("keys %v, run %+v: want one key read by both", keys, run) + } + if j, _ := r.join(step("ask-a", "a"), "main.3", "n1", "run-c"); j.started || len(j.run.Readers) != 2 { + t.Errorf("a reader joining again is added again or starts a run: %+v", j) + } + }) + t.Run("could not run is never read", func(t *testing.T) { + kv := newRunsMem() + r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf) + if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil { + t.Fatal(err) + } + key := runKeyOf("main.3", "n1") + _, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool { + c.Ended, c.Outcome = &now, meshwants.CompositionRunOutcomeCouldNotRun + return true + }) + j, err := r.join(step("ask-b", "b"), "main.3", "n1", "run-2") + if err != nil || !j.started || j.run.Ask != "run-2" || len(j.run.Readers) != 1 || j.run.Ended != nil { + t.Errorf("after a could not run: %+v %v, want a new run read by its starter alone", j, err) + } + _, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool { + c.Ended, c.Outcome = &now, meshwants.CompositionRunOutcomePassed + return true + }) + if j, _ := r.join(step("ask-c", "c"), "main.3", "n1", "run-3"); j.started || j.run.Ask != "run-2" { + t.Errorf("a passed run is not read: %+v", j) + } + if j, _ := r.join(step("ask-d", "d"), "main.4", "n1", "run-4"); !j.started { + t.Errorf("a step of another candidate reads the first's run: %+v", j) + } + }) + t.Run("cap", func(t *testing.T) { + kv := newRunsMem() + kv.bytes = 16 << 20 + r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf) + _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1") + if err == nil || !strings.Contains(err.Error(), "at its cap of 16 MiB (16777216 bytes kept)") { + t.Errorf("at the cap: %v, want the fill named", err) + } + }) + t.Run("kept", func(t *testing.T) { + kv := newRunsMem() + at := now + r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return at }, t.Logf) + for _, c := range []string{"main.1", "main.2"} { + if _, err := r.join(step("ask-"+c, "a"), c, "n1", "run-"+c); err != nil { + t.Fatal(err) + } + } + old := now.Add(-25 * time.Hour) + _, _, _ = r.change(runKeyOf("main.1", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &old, "passed"; return true }) + _ = kv.arm(runBoundPrefix+runKeyOf("main.1", "n1"), time.Minute) + recent := now.Add(-time.Hour) + _, _, _ = r.change(runKeyOf("main.2", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &recent, "passed"; return true }) + if _, err := r.join(step("ask-x", "a"), "main.9", "n1", "run-9"); err != nil { + t.Fatal(err) + } + keys, _ := kv.keys() + if want := []string{runKeyOf("main.2", "n1"), runKeyOf("main.9", "n1")}; !slices.Equal(keys, want) { + t.Errorf("keys %v, want %v: a run ended more than a day ago goes with its bound key at the next run", keys, want) + } + }) + t.Run("settings", func(t *testing.T) { + dir := t.TempDir() + files := map[string]string{"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "8", "MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": "2h", + "MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "2s", "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "", "MESH_ASSIGNER_BLAME_TRIES_FILE": "5"} + env := map[string]string{} + for name, content := range files { + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + env[name] = p + } + s, err := runSettingsOf(func(k string) string { return env[k] }) + if err != nil || s.capBytes != 8<<20 || s.kept != 2*time.Hour || s.stepBound != 2*time.Second || s.untaken != 24*time.Hour || s.blameTries != 5 { + t.Errorf("settings %+v %v", s, err) + } + _ = os.WriteFile(env["MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE"], []byte("30"), 0o600) + if _, err := runSettingsOf(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "not a positive Go duration") { + t.Errorf("a bound of no unit: %v", err) + } + if s, err := runSettingsOf(func(string) string { return "" }); err != nil || s != defaultRunSettings { + t.Errorf("no files: %+v %v, want the defaults", s, err) + } + }) + raw, _ := json.Marshal(step("x", "a")) + if !strings.Contains(string(raw), `"id":"x"`) { + t.Errorf("a reader names no ask id: %s", raw) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index e4213e17..128652db 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -169,7 +169,7 @@ func startBed(t *testing.T, leaveOut string) *bed { continue } cfg := jetstream.KeyValueConfig{Bucket: name, History: 5} - if name == plc.PulsesBucket() { + if name == plc.PulsesBucket() || name == buckets.CompositionRuns() { cfg.History, cfg.LimitMarkerTTL = 1, time.Minute // a time to live per key, as the controller makes it (key-ttl) } kv, err := js.CreateKeyValue(ctx, cfg) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 69d0fc2d..6f63dcc4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -89,7 +89,7 @@ func ownBuckets() []string { return []string{buckets.Asks(), buckets.Differences(), buckets.Puts(), buckets.Wants(), plc.AssignmentsBucket(), plc.ProposalsBucket(), plc.CandidatesBucket(), plc.PulsesBucket(), assign.FindingsBucket(), assign.PoliciesBucket(), plc.SeatHoldersBucket(), - engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.DeliveryWaits(), buckets.DeliveredAgain(), buckets.Inputs(), pinsBucket()} + engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.DeliveryWaits(), buckets.DeliveredAgain(), buckets.Inputs(), buckets.CompositionRuns(), pinsBucket()} } // engineReadsState is the assigner's state of what each node's engine states it reads (ADR 0618 §3), one key ``, as @@ -133,11 +133,12 @@ var keptStates = []announce.State{ {Name: copiesState, Readable: true}, {Name: writtenState, Readable: true}, {Name: buckets.WaitsState, Readable: true}, - {Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211) - {Name: buckets.DeliveryWaitsState, Readable: true}, // the delivery waits said (delivered.go, design 47 G A-246) - {Name: buckets.DeliveredAgainState, Readable: true}, // the asks of assigner_delivered_again (delivered.go, design 47 G A-250) - {Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354) - {Name: pinsState, Readable: true}, // the records pinning each assignment graph revision (design 47 G §579) + {Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211) + {Name: buckets.DeliveryWaitsState, Readable: true}, // the delivery waits said (delivered.go, design 47 G A-246) + {Name: buckets.DeliveredAgainState, Readable: true}, // the asks of assigner_delivered_again (delivered.go, design 47 G A-250) + {Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354) + {Name: buckets.CompositionRunsState, Readable: true}, // the composition runs, read by the process and its tools (design 47 G §719, A-468c) + {Name: pinsState, Readable: true}, // the records pinning each assignment graph revision (design 47 G §579) } // kvSet is every bucket the assigner opened, by name. @@ -682,8 +683,16 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, p.acts = newPutActs(bus, wake, logger.Printf) } // The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine. - if bus != nil && js != nil && inputs != nil && p.publishWants != nil { - run := &compositionRun{inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, + if bus != nil && js != nil && inputs != nil && p.publishWants != nil && kvs[buckets.CompositionRuns()] != nil { + set, err := runSettingsOf(getenv) + if err != nil { + return err + } + runs := newCompositionRuns(busRuns{ctx, kvs[buckets.CompositionRuns()]}, set, time.Now, logger.Printf) + if err := watchRuns(ctx, kvs[buckets.CompositionRuns()], runs, nil, nil); err != nil { + return fmt.Errorf("the state %s is not watched: %w", buckets.CompositionRunsState, err) + } + run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ends: ends, ask: bus.CallFile, now: time.Now, logf: logger.Printf} steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, now: time.Now, logf: logger.Printf, asker: stepAskerOf(seatlist.Definition(gate.Reader, meshwants.Seat))} diff --git a/modules/mesh-delivery-assigner/internal/buckets/buckets.go b/modules/mesh-delivery-assigner/internal/buckets/buckets.go index cfe1d2f6..96712348 100644 --- a/modules/mesh-delivery-assigner/internal/buckets/buckets.go +++ b/modules/mesh-delivery-assigner/internal/buckets/buckets.go @@ -29,6 +29,9 @@ const ( PutWaitsState = "put-waits" // InputsState holds the inputs of the composition steps, `