From bece2421f1e9569f6288417794cc6d6f48d983af Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 16 Nov 2026 21:31:34 +0100 Subject: [PATCH] assigner: a composition step reads its catalog revision once and again on a timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §354, A-252, A-261 --- .../compositionreads.go | 181 ++++++++++++++++++ .../mesh-delivery-assigner/compositionrun.go | 57 +++--- .../compositionrun_test.go | 83 +++++++- .../cmd/mesh-delivery-assigner/main.go | 2 +- 4 files changed, 295 insertions(+), 28 deletions(-) create mode 100644 modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionreads.go diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionreads.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionreads.go new file mode 100644 index 00000000..c4b20b41 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionreads.go @@ -0,0 +1,181 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + "time" + + mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// The reads of a composition step (design 47 G §354, A-252, A-261): the entries of one catalog revision are read once and shared by +// every composition step of that revision, since an entry is content-addressed and a revision never changes; a read that times out +// or fails in transport is made again with a growing wait for a bounded time, said in the journal, and only then ends the step as +// could not run. + +const ( + // readsBound is how long a read is made again before the step could not run. + readsBound = 3 * time.Minute + // readsFirstWait and readsMaxWait bound the wait between two tries. + readsFirstWait = time.Second + readsMaxWait = 15 * time.Second + // readsKept is how many catalog revisions' halves are kept; the oldest read goes first. + readsKept = 8 +) + +// readKey names one revision of one watched branch. +type readKey struct { + branch string + revision int +} + +// readSlot is one revision's read: done closes when half or err is set. +type readSlot struct { + done chan struct{} + half mcc.Half + err error +} + +// compositionReads shares the reads of composition steps. +type compositionReads struct { + mu sync.Mutex + revs map[readKey]*readSlot + order []readKey + seats map[string]json.RawMessage // by seat and digest: the definition, raw +} + +// transient says whether a read's fault is of the transport or a timeout, so that it is worth reading again. +func transient(err error, said string) bool { + text := said + if err != nil { + text = err.Error() + " " + said + } + text = strings.ToLower(text) + for _, w := range []string{"deadline exceeded", "timeout", "timed out", "no responders", "connection", "i/o", "eof", "broken pipe", "unavailable"} { + if strings.Contains(text, w) { + return true + } + } + return false +} + +// callRetry calls the seat verb key, again with a growing wait while its fault is transient, for readsBound; it answers the body of +// a good answer, or the last fault. +func (r *compositionRun) callRetry(ctx context.Context, key string, body any) (json.RawMessage, string, error) { + start := r.now() + wait := readsFirstWait + for try := 1; ; try++ { + raw, err := r.call(key, body) + var inner json.RawMessage + var said string + if err == nil { + var ok bool + if inner, said, ok = buscalls.Unwrap(raw); ok { + return inner, "", nil + } + } + if !transient(err, said) { + return nil, said, err + } + if r.now().Sub(start)+wait > readsBound { + return nil, said, err + } + r.logf("composition step: %s is not read (try %d, %v %s), read again in %s", key, try, err, said, wait) + if serr := r.waitFor(ctx, wait); serr != nil { + return nil, said, serr + } + if wait *= 2; wait > readsMaxWait { + wait = readsMaxWait + } + } +} + +// waitFor waits d, or until ctx ends. +func (r *compositionRun) waitFor(ctx context.Context, d time.Duration) error { + if r.sleep != nil { + return r.sleep(ctx, d) + } + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return nil + } +} + +// half is the revision's half, read once for every step of it: a step that finds it being read waits for that read, and a read that +// failed is not kept, so the next step reads again. +func (r *compositionRun) half(ctx context.Context, key readKey, read func() (mcc.Half, error)) (mcc.Half, error) { + c := r.reads + c.mu.Lock() + if c.revs == nil { + c.revs = map[readKey]*readSlot{} + } + if s, ok := c.revs[key]; ok { + c.mu.Unlock() + select { + case <-s.done: + if s.err == nil { + return s.half, nil + } + case <-ctx.Done(): + return mcc.Half{}, ctx.Err() + } + // The read this step waited for failed: read it here (the failed slot is already dropped). + return r.half(ctx, key, read) + } + s := &readSlot{done: make(chan struct{})} + c.revs[key] = s + c.order = append(c.order, key) + for len(c.order) > readsKept { + delete(c.revs, c.order[0]) + c.order = c.order[1:] + } + c.mu.Unlock() + s.half, s.err = read() + if s.err != nil { + c.mu.Lock() + if c.revs[key] == s { + delete(c.revs, key) + } + for i, k := range c.order { + if k == key { + c.order = append(c.order[:i], c.order[i+1:]...) + break + } + } + c.mu.Unlock() + } + close(s.done) + return s.half, s.err +} + +// seatKept is the definition kept of a seat at a digest. +func (c *compositionReads) seatKept(seat, digest string) (json.RawMessage, bool) { + c.mu.Lock() + defer c.mu.Unlock() + d, ok := c.seats[seat+"@"+digest] + return d, ok +} + +func (c *compositionReads) keepSeat(seat, digest string, def json.RawMessage) { + c.mu.Lock() + defer c.mu.Unlock() + if c.seats == nil { + c.seats = map[string]json.RawMessage{} + } + c.seats[seat+"@"+digest] = def +} + +func unreadErr(what string, err error, said string) error { + if err != nil { + return fmt.Errorf("%s: %w", what, err) + } + return fmt.Errorf("%s: %s", what, said) +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go index 3abf6f12..5d98c8cd 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go @@ -18,7 +18,6 @@ import ( meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" - "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/keys" ) @@ -122,6 +121,9 @@ type compositionRun struct { // put puts a run ask on the build seat's check queue under the assigner's name. put func(ctx context.Context, a buildstep.StepAsk) error ends *runEnds + // reads shares the catalog reads among steps (compositionreads.go); sleep waits between two tries of a read (nil: a timer). + reads *compositionReads + sleep func(ctx context.Context, d time.Duration) error // ask asks a seat verb on one node, for the run's answer kept as the build seat's object (buildstep.StepEnded.ReadOutput). ask buildstep.AskVerb now func() time.Time @@ -136,11 +138,11 @@ type machineEnd struct { // run runs one composition step and answers what it came to: failed when a machine does not compose, could not run when a machine // could not be composed, else passed with a warning per machine away or not judged. func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRun) stepOutcome { - half, err := r.revision(run) + half, err := r.revision(ctx, run) if err != nil { return couldNotRun("the step's catalog revision is not read: " + err.Error()) } - seats, err := r.seats(half.Revision.Definitions) + seats, err := r.seats(ctx, half.Revision.Definitions) if err != nil { return couldNotRun(err.Error()) } @@ -148,7 +150,7 @@ func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRu if err != nil { return couldNotRun(err.Error()) } - programOf, err := r.programs(half) + programOf, err := r.programs(ctx, half) if err != nil { return couldNotRun(err.Error()) } @@ -275,20 +277,24 @@ func verdictOf(node string, answer []byte) machineEnd { return machineEnd{couldNot: node + ": the composer's verdict has no case " + changeCase} } -// revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252). -func (r *compositionRun) revision(run meshwants.StepRun) (mcc.Half, error) { +// revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252), once for every step of +// it (compositionreads.go). +func (r *compositionRun) revision(ctx context.Context, run meshwants.StepRun) (mcc.Half, error) { if run.Revision <= 0 { return mcc.Half{}, errors.New("the step names no catalog revision") } + return r.half(ctx, readKey{run.Branch, int(run.Revision)}, func() (mcc.Half, error) { return r.readRevision(ctx, run) }) +} + +func (r *compositionRun) readRevision(ctx context.Context, run meshwants.StepRun) (mcc.Half, error) { half := mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}} in := meshcatalog.RevisionInput{Branch: keys.EscapeName(run.Branch), Revision: run.Revision} for { - raw, err := r.call(mcc.RevisionVerb, in) + body, said, err := r.callRetry(ctx, mcc.RevisionVerb, in) if err != nil { return mcc.Half{}, err } - body, said, ok := buscalls.Unwrap(raw) - if !ok { + if body == nil { return mcc.Half{}, fmt.Errorf("%s answered: %s", mcc.RevisionVerb, said) } var a meshcatalog.RevisionAnswer @@ -314,20 +320,21 @@ func (r *compositionRun) revision(run meshwants.StepRun) (mcc.Half, error) { // seats are the definitions of the revision's (seat, digest) pairs, raw (A-261): the revision's definitions carry no definition // yet (A-234), so each is mesh-catalog's seat-definitions answer for the seat where the trunk's digest is the revision's; a digest // neither holds is an error naming the seat and the digest. -func (r *compositionRun) seats(defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, error) { +func (r *compositionRun) seats(ctx context.Context, defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, error) { out := make([]json.RawMessage, 0, len(defs)) for _, d := range defs { - raw, err := r.call(seatDefinitionsKey, meshcatalog.SeatDefinitionsInput{Seat: d.Seat}) - if err != nil { - return nil, fmt.Errorf("the definition of the seat %s is not read: %w", d.Seat, err) + if def, ok := r.reads.seatKept(d.Seat, d.Digest); ok { // a definition at a digest never changes + out = append(out, def) + continue } - body, said, ok := buscalls.Unwrap(raw) - if !ok { - return nil, fmt.Errorf("the definition of the seat %s is not read: %s", d.Seat, said) + what := "the definition of the seat " + d.Seat + " is not read" + body, said, err := r.callRetry(ctx, seatDefinitionsKey, meshcatalog.SeatDefinitionsInput{Seat: d.Seat}) + if err != nil || body == nil { + return nil, unreadErr(what, err, said) } var a meshcatalog.SeatDefinitionsAnswer if err := json.Unmarshal(body, &a); err != nil { - return nil, fmt.Errorf("the definition of the seat %s is not read: %w", d.Seat, err) + return nil, fmt.Errorf("%s: %w", what, err) } if a.Entry == nil || a.Entry.Digest != d.Digest || a.Entry.Definition == nil { return nil, fmt.Errorf("no definition of the seat %s with the digest %s is held: neither the revision nor the trunk holds it", d.Seat, d.Digest) @@ -336,6 +343,7 @@ func (r *compositionRun) seats(defs []meshcatalog.CatalogRevisionDefinitionsItem if err != nil { return nil, err } + r.reads.keepSeat(d.Seat, d.Digest, def) out = append(out, def) } return out, nil @@ -412,7 +420,7 @@ func programOfEntry(e meshcatalog.CatalogEntry) (buildstep.RunProgram, error) { // programs answers, per machine, the composer its run builds (A-226, A-243, A-252): where the revision's node-engine entry differs // from the trunk's, that build for every machine; else the build the machine's node-engine runs, as its `running` says, its entry // read through mesh-catalog's `entry`; a machine whose node-engine's `running` is not heard gets the revision's entry, said (A-451). -func (r *compositionRun) programs(half mcc.Half) (func(node string) (buildstep.RunProgram, error), error) { +func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, error), error) { engine, ok := nodeEngineEntry(half.Entries) if !ok { return nil, fmt.Errorf("the catalog revision %d holds no module claiming the seat %s", half.Revision.Revision, ne.Seat) @@ -436,17 +444,14 @@ func (r *compositionRun) programs(half mcc.Half) (func(node string) (buildstep.R if ok { return p, nil } - raw, err := r.call(entryVerbKey, meshcatalog.EntryInput{Fingerprint: running.Fingerprint}) - if err != nil { - return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %w", running.Fingerprint, err) - } - body, said, ok := buscalls.Unwrap(raw) - if !ok { - return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %s", running.Fingerprint, said) + what := "the entry of the node-engine's build " + running.Fingerprint + " is not read" + body, said, err := r.callRetry(ctx, entryVerbKey, meshcatalog.EntryInput{Fingerprint: running.Fingerprint}) + if err != nil || body == nil { + return buildstep.RunProgram{}, unreadErr(what, err, said) } var a meshcatalog.EntryAnswer if err := json.Unmarshal(body, &a); err != nil { - return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %w", running.Fingerprint, err) + return buildstep.RunProgram{}, fmt.Errorf("%s: %w", what, err) } p, err = programOfEntry(a.Entry) if err != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go index fe99d51d..c9ccd69e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go @@ -82,7 +82,7 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) * ends := &runEnds{} s.r = &compositionRun{ inputs: &compositionInputs{machines: func(string, string) []string { return []string{"ace", "g14", "shanks"} }, callFile: engines, bound: time.Second}, - keeper: keeper, call: call, ends: ends, away: away, now: time.Now, logf: t.Logf, + keeper: keeper, reads: &compositionReads{}, call: call, ends: ends, away: away, now: time.Now, logf: t.Logf, trunk: func() (mcc.Half, bool) { return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{"engine": engine(trunkEngine, commitE1)}}, true }, @@ -187,3 +187,84 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) { t.Errorf("an ask at the cap came to %+v, %d asks", out, len(full.asks)) } } + +// Design 47 G §354, A-252: the entries of one catalog revision are read once for every step of it, and a read that times out is +// made again with a growing wait for a bounded time, said in the journal, before the step could not run. +func TestTheCompositionStepReadsTheRevisionOnceAndAgainOnATimeout(t *testing.T) { + step := meshwants.StepRun{Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition} + for _, tc := range []struct { + name string + faults int // consecutive faults of the `revision` read before it answers + fault error // the fault + steps int + want string // the outcome of the last step + revisionCalls int + waits int + }{ + {"two steps of one revision read it once", 0, nil, 2, meshwants.StepEndedOutcomeFailed, 2, 0}, + {"a timeout is read again", 3, errors.New("context deadline exceeded"), 1, meshwants.StepEndedOutcomeFailed, 5, 3}, + {"a fault that stays ends could not run", 1000, errors.New("nats: timeout"), 1, meshwants.StepEndedOutcomeCouldNotRun, -1, -1}, + {"a fault of the answer is not read again", 1, errors.New("no such revision"), 1, meshwants.StepEndedOutcomeCouldNotRun, 1, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + s := newRunStand(t, "src2:e1", "src2:e1", "d1") + inner := s.r.call + var mu sync.Mutex + calls, faults := 0, tc.faults + s.r.call = func(key string, body any) (json.RawMessage, error) { + if key == mcc.RevisionVerb { + mu.Lock() + calls++ + f := faults > 0 + faults-- + mu.Unlock() + if f { + return nil, tc.fault + } + } + return inner(key, body) + } + clock := time.Unix(1000, 0) + var waits []time.Duration + s.r.now = func() time.Time { mu.Lock(); defer mu.Unlock(); return clock } + s.r.sleep = func(_ context.Context, d time.Duration) error { + mu.Lock() + defer mu.Unlock() + waits = append(waits, d) + clock = clock.Add(d) + return nil + } + var wg sync.WaitGroup + outs := make([]stepOutcome, tc.steps) + for i := range outs { + wg.Add(1) + go func() { defer wg.Done(); outs[i] = s.r.run(context.Background(), "ask-step", step) }() + } + wg.Wait() + for _, o := range outs { + if o.Outcome != tc.want { + t.Fatalf("a step came to %+v, want %s", o, tc.want) + } + } + if tc.waits == -1 { // never answers: bounded, growing waits + if len(waits) < 3 || waits[0] != readsFirstWait || waits[1] != 2*readsFirstWait || waits[len(waits)-1] != readsMaxWait { + t.Errorf("the waits %v do not grow to %s", waits, readsMaxWait) + } + var total time.Duration + for _, w := range waits { + total += w + } + if total > readsBound { + t.Errorf("read again for %s, bound %s", total, readsBound) + } + if !strings.Contains(outs[0].Said, "catalog revision is not read") || !strings.Contains(outs[0].Said, "timeout") { + t.Errorf("the step said %q", outs[0].Said) + } + return + } + if calls != tc.revisionCalls || len(waits) != tc.waits { + t.Errorf("`revision` asked %d times with %d waits, want %d and %d", calls, len(waits), tc.revisionCalls, tc.waits) + } + }) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 7a6b359b..fbff5f7f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -675,7 +675,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } // The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine. if bus != nil && js != nil && inputs != nil && p.publishWants != nil { - run := &compositionRun{inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, call: bus.CallKept, + run := &compositionRun{inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ends: ends, ask: bus.CallFile, now: time.Now, logf: logger.Printf} steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, now: time.Now, logf: logger.Printf, asker: stepAskerOf(seatlist.Definition(gate.Reader, meshwants.Seat))} -- 2.54.0