assigner: a composition step reads its catalog revision once and again on a timeout #232
@@ -0,0 +1,181 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
)
|
||||
|
||||
// The reads of a composition step (design 47 G §354, A-252, A-261): the entries of one catalog revision are read once and shared by
|
||||
// every composition step of that revision, since an entry is content-addressed and a revision never changes; a read that times out
|
||||
// or fails in transport is made again with a growing wait for a bounded time, said in the journal, and only then ends the step as
|
||||
// could not run.
|
||||
|
||||
const (
|
||||
// readsBound is how long a read is made again before the step could not run.
|
||||
readsBound = 3 * time.Minute
|
||||
// readsFirstWait and readsMaxWait bound the wait between two tries.
|
||||
readsFirstWait = time.Second
|
||||
readsMaxWait = 15 * time.Second
|
||||
// readsKept is how many catalog revisions' halves are kept; the oldest read goes first.
|
||||
readsKept = 8
|
||||
)
|
||||
|
||||
// readKey names one revision of one watched branch.
|
||||
type readKey struct {
|
||||
branch string
|
||||
revision int
|
||||
}
|
||||
|
||||
// readSlot is one revision's read: done closes when half or err is set.
|
||||
type readSlot struct {
|
||||
done chan struct{}
|
||||
half mcc.Half
|
||||
err error
|
||||
}
|
||||
|
||||
// compositionReads shares the reads of composition steps.
|
||||
type compositionReads struct {
|
||||
mu sync.Mutex
|
||||
revs map[readKey]*readSlot
|
||||
order []readKey
|
||||
seats map[string]json.RawMessage // by seat and digest: the definition, raw
|
||||
}
|
||||
|
||||
// transient says whether a read's fault is of the transport or a timeout, so that it is worth reading again.
|
||||
func transient(err error, said string) bool {
|
||||
text := said
|
||||
if err != nil {
|
||||
text = err.Error() + " " + said
|
||||
}
|
||||
text = strings.ToLower(text)
|
||||
for _, w := range []string{"deadline exceeded", "timeout", "timed out", "no responders", "connection", "i/o", "eof", "broken pipe", "unavailable"} {
|
||||
if strings.Contains(text, w) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// callRetry calls the seat verb key, again with a growing wait while its fault is transient, for readsBound; it answers the body of
|
||||
// a good answer, or the last fault.
|
||||
func (r *compositionRun) callRetry(ctx context.Context, key string, body any) (json.RawMessage, string, error) {
|
||||
start := r.now()
|
||||
wait := readsFirstWait
|
||||
for try := 1; ; try++ {
|
||||
raw, err := r.call(key, body)
|
||||
var inner json.RawMessage
|
||||
var said string
|
||||
if err == nil {
|
||||
var ok bool
|
||||
if inner, said, ok = buscalls.Unwrap(raw); ok {
|
||||
return inner, "", nil
|
||||
}
|
||||
}
|
||||
if !transient(err, said) {
|
||||
return nil, said, err
|
||||
}
|
||||
if r.now().Sub(start)+wait > readsBound {
|
||||
return nil, said, err
|
||||
}
|
||||
r.logf("composition step: %s is not read (try %d, %v %s), read again in %s", key, try, err, said, wait)
|
||||
if serr := r.waitFor(ctx, wait); serr != nil {
|
||||
return nil, said, serr
|
||||
}
|
||||
if wait *= 2; wait > readsMaxWait {
|
||||
wait = readsMaxWait
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// waitFor waits d, or until ctx ends.
|
||||
func (r *compositionRun) waitFor(ctx context.Context, d time.Duration) error {
|
||||
if r.sleep != nil {
|
||||
return r.sleep(ctx, d)
|
||||
}
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-t.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// half is the revision's half, read once for every step of it: a step that finds it being read waits for that read, and a read that
|
||||
// failed is not kept, so the next step reads again.
|
||||
func (r *compositionRun) half(ctx context.Context, key readKey, read func() (mcc.Half, error)) (mcc.Half, error) {
|
||||
c := r.reads
|
||||
c.mu.Lock()
|
||||
if c.revs == nil {
|
||||
c.revs = map[readKey]*readSlot{}
|
||||
}
|
||||
if s, ok := c.revs[key]; ok {
|
||||
c.mu.Unlock()
|
||||
select {
|
||||
case <-s.done:
|
||||
if s.err == nil {
|
||||
return s.half, nil
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return mcc.Half{}, ctx.Err()
|
||||
}
|
||||
// The read this step waited for failed: read it here (the failed slot is already dropped).
|
||||
return r.half(ctx, key, read)
|
||||
}
|
||||
s := &readSlot{done: make(chan struct{})}
|
||||
c.revs[key] = s
|
||||
c.order = append(c.order, key)
|
||||
for len(c.order) > readsKept {
|
||||
delete(c.revs, c.order[0])
|
||||
c.order = c.order[1:]
|
||||
}
|
||||
c.mu.Unlock()
|
||||
s.half, s.err = read()
|
||||
if s.err != nil {
|
||||
c.mu.Lock()
|
||||
if c.revs[key] == s {
|
||||
delete(c.revs, key)
|
||||
}
|
||||
for i, k := range c.order {
|
||||
if k == key {
|
||||
c.order = append(c.order[:i], c.order[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
close(s.done)
|
||||
return s.half, s.err
|
||||
}
|
||||
|
||||
// seatKept is the definition kept of a seat at a digest.
|
||||
func (c *compositionReads) seatKept(seat, digest string) (json.RawMessage, bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
d, ok := c.seats[seat+"@"+digest]
|
||||
return d, ok
|
||||
}
|
||||
|
||||
func (c *compositionReads) keepSeat(seat, digest string, def json.RawMessage) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.seats == nil {
|
||||
c.seats = map[string]json.RawMessage{}
|
||||
}
|
||||
c.seats[seat+"@"+digest] = def
|
||||
}
|
||||
|
||||
func unreadErr(what string, err error, said string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
return fmt.Errorf("%s: %s", what, said)
|
||||
}
|
||||
@@ -18,7 +18,6 @@ import (
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
@@ -122,6 +121,9 @@ type compositionRun struct {
|
||||
// put puts a run ask on the build seat's check queue under the assigner's name.
|
||||
put func(ctx context.Context, a buildstep.StepAsk) error
|
||||
ends *runEnds
|
||||
// reads shares the catalog reads among steps (compositionreads.go); sleep waits between two tries of a read (nil: a timer).
|
||||
reads *compositionReads
|
||||
sleep func(ctx context.Context, d time.Duration) error
|
||||
// ask asks a seat verb on one node, for the run's answer kept as the build seat's object (buildstep.StepEnded.ReadOutput).
|
||||
ask buildstep.AskVerb
|
||||
now func() time.Time
|
||||
@@ -136,11 +138,11 @@ type machineEnd struct {
|
||||
// run runs one composition step and answers what it came to: failed when a machine does not compose, could not run when a machine
|
||||
// could not be composed, else passed with a warning per machine away or not judged.
|
||||
func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRun) stepOutcome {
|
||||
half, err := r.revision(run)
|
||||
half, err := r.revision(ctx, run)
|
||||
if err != nil {
|
||||
return couldNotRun("the step's catalog revision is not read: " + err.Error())
|
||||
}
|
||||
seats, err := r.seats(half.Revision.Definitions)
|
||||
seats, err := r.seats(ctx, half.Revision.Definitions)
|
||||
if err != nil {
|
||||
return couldNotRun(err.Error())
|
||||
}
|
||||
@@ -148,7 +150,7 @@ func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRu
|
||||
if err != nil {
|
||||
return couldNotRun(err.Error())
|
||||
}
|
||||
programOf, err := r.programs(half)
|
||||
programOf, err := r.programs(ctx, half)
|
||||
if err != nil {
|
||||
return couldNotRun(err.Error())
|
||||
}
|
||||
@@ -275,20 +277,24 @@ func verdictOf(node string, answer []byte) machineEnd {
|
||||
return machineEnd{couldNot: node + ": the composer's verdict has no case " + changeCase}
|
||||
}
|
||||
|
||||
// revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252).
|
||||
func (r *compositionRun) revision(run meshwants.StepRun) (mcc.Half, error) {
|
||||
// revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252), once for every step of
|
||||
// it (compositionreads.go).
|
||||
func (r *compositionRun) revision(ctx context.Context, run meshwants.StepRun) (mcc.Half, error) {
|
||||
if run.Revision <= 0 {
|
||||
return mcc.Half{}, errors.New("the step names no catalog revision")
|
||||
}
|
||||
return r.half(ctx, readKey{run.Branch, int(run.Revision)}, func() (mcc.Half, error) { return r.readRevision(ctx, run) })
|
||||
}
|
||||
|
||||
func (r *compositionRun) readRevision(ctx context.Context, run meshwants.StepRun) (mcc.Half, error) {
|
||||
half := mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}}
|
||||
in := meshcatalog.RevisionInput{Branch: keys.EscapeName(run.Branch), Revision: run.Revision}
|
||||
for {
|
||||
raw, err := r.call(mcc.RevisionVerb, in)
|
||||
body, said, err := r.callRetry(ctx, mcc.RevisionVerb, in)
|
||||
if err != nil {
|
||||
return mcc.Half{}, err
|
||||
}
|
||||
body, said, ok := buscalls.Unwrap(raw)
|
||||
if !ok {
|
||||
if body == nil {
|
||||
return mcc.Half{}, fmt.Errorf("%s answered: %s", mcc.RevisionVerb, said)
|
||||
}
|
||||
var a meshcatalog.RevisionAnswer
|
||||
@@ -314,20 +320,21 @@ func (r *compositionRun) revision(run meshwants.StepRun) (mcc.Half, error) {
|
||||
// seats are the definitions of the revision's (seat, digest) pairs, raw (A-261): the revision's definitions carry no definition
|
||||
// yet (A-234), so each is mesh-catalog's seat-definitions answer for the seat where the trunk's digest is the revision's; a digest
|
||||
// neither holds is an error naming the seat and the digest.
|
||||
func (r *compositionRun) seats(defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, error) {
|
||||
func (r *compositionRun) seats(ctx context.Context, defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, error) {
|
||||
out := make([]json.RawMessage, 0, len(defs))
|
||||
for _, d := range defs {
|
||||
raw, err := r.call(seatDefinitionsKey, meshcatalog.SeatDefinitionsInput{Seat: d.Seat})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the definition of the seat %s is not read: %w", d.Seat, err)
|
||||
if def, ok := r.reads.seatKept(d.Seat, d.Digest); ok { // a definition at a digest never changes
|
||||
out = append(out, def)
|
||||
continue
|
||||
}
|
||||
body, said, ok := buscalls.Unwrap(raw)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the definition of the seat %s is not read: %s", d.Seat, said)
|
||||
what := "the definition of the seat " + d.Seat + " is not read"
|
||||
body, said, err := r.callRetry(ctx, seatDefinitionsKey, meshcatalog.SeatDefinitionsInput{Seat: d.Seat})
|
||||
if err != nil || body == nil {
|
||||
return nil, unreadErr(what, err, said)
|
||||
}
|
||||
var a meshcatalog.SeatDefinitionsAnswer
|
||||
if err := json.Unmarshal(body, &a); err != nil {
|
||||
return nil, fmt.Errorf("the definition of the seat %s is not read: %w", d.Seat, err)
|
||||
return nil, fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
if a.Entry == nil || a.Entry.Digest != d.Digest || a.Entry.Definition == nil {
|
||||
return nil, fmt.Errorf("no definition of the seat %s with the digest %s is held: neither the revision nor the trunk holds it", d.Seat, d.Digest)
|
||||
@@ -336,6 +343,7 @@ func (r *compositionRun) seats(defs []meshcatalog.CatalogRevisionDefinitionsItem
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.reads.keepSeat(d.Seat, d.Digest, def)
|
||||
out = append(out, def)
|
||||
}
|
||||
return out, nil
|
||||
@@ -412,7 +420,7 @@ func programOfEntry(e meshcatalog.CatalogEntry) (buildstep.RunProgram, error) {
|
||||
// programs answers, per machine, the composer its run builds (A-226, A-243, A-252): where the revision's node-engine entry differs
|
||||
// from the trunk's, that build for every machine; else the build the machine's node-engine runs, as its `running` says, its entry
|
||||
// read through mesh-catalog's `entry`; a machine whose node-engine's `running` is not heard gets the revision's entry, said (A-451).
|
||||
func (r *compositionRun) programs(half mcc.Half) (func(node string) (buildstep.RunProgram, error), error) {
|
||||
func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, error), error) {
|
||||
engine, ok := nodeEngineEntry(half.Entries)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the catalog revision %d holds no module claiming the seat %s", half.Revision.Revision, ne.Seat)
|
||||
@@ -436,17 +444,14 @@ func (r *compositionRun) programs(half mcc.Half) (func(node string) (buildstep.R
|
||||
if ok {
|
||||
return p, nil
|
||||
}
|
||||
raw, err := r.call(entryVerbKey, meshcatalog.EntryInput{Fingerprint: running.Fingerprint})
|
||||
if err != nil {
|
||||
return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %w", running.Fingerprint, err)
|
||||
}
|
||||
body, said, ok := buscalls.Unwrap(raw)
|
||||
if !ok {
|
||||
return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %s", running.Fingerprint, said)
|
||||
what := "the entry of the node-engine's build " + running.Fingerprint + " is not read"
|
||||
body, said, err := r.callRetry(ctx, entryVerbKey, meshcatalog.EntryInput{Fingerprint: running.Fingerprint})
|
||||
if err != nil || body == nil {
|
||||
return buildstep.RunProgram{}, unreadErr(what, err, said)
|
||||
}
|
||||
var a meshcatalog.EntryAnswer
|
||||
if err := json.Unmarshal(body, &a); err != nil {
|
||||
return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %w", running.Fingerprint, err)
|
||||
return buildstep.RunProgram{}, fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
p, err = programOfEntry(a.Entry)
|
||||
if err != nil {
|
||||
|
||||
@@ -82,7 +82,7 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
|
||||
ends := &runEnds{}
|
||||
s.r = &compositionRun{
|
||||
inputs: &compositionInputs{machines: func(string, string) []string { return []string{"ace", "g14", "shanks"} }, callFile: engines, bound: time.Second},
|
||||
keeper: keeper, call: call, ends: ends, away: away, now: time.Now, logf: t.Logf,
|
||||
keeper: keeper, reads: &compositionReads{}, call: call, ends: ends, away: away, now: time.Now, logf: t.Logf,
|
||||
trunk: func() (mcc.Half, bool) {
|
||||
return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{"engine": engine(trunkEngine, commitE1)}}, true
|
||||
},
|
||||
@@ -187,3 +187,84 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
|
||||
t.Errorf("an ask at the cap came to %+v, %d asks", out, len(full.asks))
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §354, A-252: the entries of one catalog revision are read once for every step of it, and a read that times out is
|
||||
// made again with a growing wait for a bounded time, said in the journal, before the step could not run.
|
||||
func TestTheCompositionStepReadsTheRevisionOnceAndAgainOnATimeout(t *testing.T) {
|
||||
step := meshwants.StepRun{Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
faults int // consecutive faults of the `revision` read before it answers
|
||||
fault error // the fault
|
||||
steps int
|
||||
want string // the outcome of the last step
|
||||
revisionCalls int
|
||||
waits int
|
||||
}{
|
||||
{"two steps of one revision read it once", 0, nil, 2, meshwants.StepEndedOutcomeFailed, 2, 0},
|
||||
{"a timeout is read again", 3, errors.New("context deadline exceeded"), 1, meshwants.StepEndedOutcomeFailed, 5, 3},
|
||||
{"a fault that stays ends could not run", 1000, errors.New("nats: timeout"), 1, meshwants.StepEndedOutcomeCouldNotRun, -1, -1},
|
||||
{"a fault of the answer is not read again", 1, errors.New("no such revision"), 1, meshwants.StepEndedOutcomeCouldNotRun, 1, 0},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
inner := s.r.call
|
||||
var mu sync.Mutex
|
||||
calls, faults := 0, tc.faults
|
||||
s.r.call = func(key string, body any) (json.RawMessage, error) {
|
||||
if key == mcc.RevisionVerb {
|
||||
mu.Lock()
|
||||
calls++
|
||||
f := faults > 0
|
||||
faults--
|
||||
mu.Unlock()
|
||||
if f {
|
||||
return nil, tc.fault
|
||||
}
|
||||
}
|
||||
return inner(key, body)
|
||||
}
|
||||
clock := time.Unix(1000, 0)
|
||||
var waits []time.Duration
|
||||
s.r.now = func() time.Time { mu.Lock(); defer mu.Unlock(); return clock }
|
||||
s.r.sleep = func(_ context.Context, d time.Duration) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
waits = append(waits, d)
|
||||
clock = clock.Add(d)
|
||||
return nil
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
outs := make([]stepOutcome, tc.steps)
|
||||
for i := range outs {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); outs[i] = s.r.run(context.Background(), "ask-step", step) }()
|
||||
}
|
||||
wg.Wait()
|
||||
for _, o := range outs {
|
||||
if o.Outcome != tc.want {
|
||||
t.Fatalf("a step came to %+v, want %s", o, tc.want)
|
||||
}
|
||||
}
|
||||
if tc.waits == -1 { // never answers: bounded, growing waits
|
||||
if len(waits) < 3 || waits[0] != readsFirstWait || waits[1] != 2*readsFirstWait || waits[len(waits)-1] != readsMaxWait {
|
||||
t.Errorf("the waits %v do not grow to %s", waits, readsMaxWait)
|
||||
}
|
||||
var total time.Duration
|
||||
for _, w := range waits {
|
||||
total += w
|
||||
}
|
||||
if total > readsBound {
|
||||
t.Errorf("read again for %s, bound %s", total, readsBound)
|
||||
}
|
||||
if !strings.Contains(outs[0].Said, "catalog revision is not read") || !strings.Contains(outs[0].Said, "timeout") {
|
||||
t.Errorf("the step said %q", outs[0].Said)
|
||||
}
|
||||
return
|
||||
}
|
||||
if calls != tc.revisionCalls || len(waits) != tc.waits {
|
||||
t.Errorf("`revision` asked %d times with %d waits, want %d and %d", calls, len(waits), tc.revisionCalls, tc.waits)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -675,7 +675,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
// The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine.
|
||||
if bus != nil && js != nil && inputs != nil && p.publishWants != nil {
|
||||
run := &compositionRun{inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, call: bus.CallKept,
|
||||
run := &compositionRun{inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept,
|
||||
trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ends: ends, ask: bus.CallFile, now: time.Now, logf: logger.Printf}
|
||||
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, now: time.Now, logf: logger.Printf,
|
||||
asker: stepAskerOf(seatlist.Definition(gate.Reader, meshwants.Seat))}
|
||||
|
||||
Reference in New Issue
Block a user