diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go index 52c522630..afb8b1fb2 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go @@ -15,6 +15,8 @@ import ( nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" ) // The inputs of a composition run (design 47 G §354, §718, A-243, A-252): one request per candidate and machine, carrying that @@ -72,28 +74,44 @@ type nodeInput struct { // compositionInputs asks the machines of a step for their inputs. type compositionInputs struct { machines func(module, branch string) []string - // callFile asks a seat verb marked chunks, addressed to one node as `seat:.@`, and answers the verb's answer with - // the whole file received added as its member `file`, base64 (process.Conn.CallFile; design 47 G A-237b, A-237c, A-249). The call - // token is the body's `call`. - callFile func(key string, body any) (json.RawMessage, error) - bound time.Duration + // callFile asks a seat verb marked chunks as a kept call (process.Conn.CallFileKept; design 47 H2 §775, H-A-405), addressed to + // one node as `seat:.@`, carrying the ask's id and stopping at its bound, and answers the verb's answer with + // the whole file received added as its member `file`, base64 (design 47 G A-237b, A-237c, A-249). The call token is the + // body's `call`. + callFile func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) + // bound is compose-ask-bound: how long a machine's compose-candidate is asked again through a transport fault (§775). + bound time.Duration } -// of asks the machine node for its input with the candidate's key (§718): asked only by the composition step that starts the -// machine's composition run, once per candidate and machine. A node that is silent, unready or answers no input has Why set. -func (c *compositionInputs) of(ctx context.Context, node, candidate string) nodeInput { +// composeAskBoundSetting is the holder's bound on its compose-candidate asks (design 47 H2 §775): past it the machine gave no +// input, naming every call. +const ( + composeAskBoundSetting = "compose-ask-bound" + defaultComposeAskBound = 10 * time.Second +) + +// composeAskBound reads the setting compose-ask-bound (§781). +func composeAskBound() (time.Duration, error) { + d, err := stdio.SettingDuration(composeAskBoundSetting, defaultComposeAskBound) + if err != nil { + return 0, errors.New("the assigner's " + err.Error()) + } + return d, nil +} + +// composeAskID is the id of the compose-candidate ask of the step asked under stepAsk to node (H-A-405 (2)): one per node per +// step, so the engines' mesh-wide state taken keeps each apart, and a repeat to the same node carries the same id. +func composeAskID(stepAsk, node string) string { return stepAsk + "." + node } + +// of asks the machine node for its input with the candidate's key (§718), as a kept call under the id of stepAsk, the step +// that starts the machine's composition run: asked once per candidate and machine, again only through a transport fault until +// compose-ask-bound (§775). A node that is silent past the bound, unready or answers no input has Why set. +func (c *compositionInputs) of(ctx context.Context, node, candidate, stepAsk string) nodeInput { bound := c.bound if bound <= 0 { - bound = nec.AnswerBound + bound = defaultComposeAskBound } - return c.one(ctx, bound, node, candidate) -} - -// one asks one node for its input. -func (c *compositionInputs) one(ctx context.Context, bound time.Duration, node, key string) nodeInput { in := nodeInput{Node: node} - at, cancel := context.WithTimeout(ctx, bound) - defer cancel() type reply struct { raw json.RawMessage err error @@ -106,13 +124,14 @@ func (c *compositionInputs) one(ctx context.Context, bound time.Duration, node, got <- reply{nil, err} return } - raw, err := c.callFile(composeCandidateKey+"@"+node, ne.ComposeCandidateInput{Candidate: key, Input: true, Call: hex.EncodeToString(bits[:])}) + raw, err := c.callFile(process.KeptAsk{ID: composeAskID(stepAsk, node), Bound: bound}, composeCandidateKey+"@"+node, + ne.ComposeCandidateInput{Candidate: candidate, Input: true, Call: hex.EncodeToString(bits[:])}) got <- reply{raw, err} }() var raw json.RawMessage select { - case <-at.Done(): - in.Why = "no answer within its bound: " + at.Err().Error() + case <-ctx.Done(): + in.Why = "its step's wait ended: " + context.Cause(ctx).Error() return in case r := <-got: if r.err != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go index d403de527..596bcbab5 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "errors" + "git.novox.be/novox/mesh-sdk/go/process" "regexp" "strings" "sync" @@ -71,11 +72,12 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) { // the machines are asked side by side, so the record of what each was asked is guarded. var mu sync.Mutex asked := map[string]string{} - call := func(key string, body any) (json.RawMessage, error) { + asks := map[string]process.KeptAsk{} + call := func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) { raw, _ := json.Marshal(body) key, node, _ := strings.Cut(key, "@") // addressed to one node (A-237b) mu.Lock() - asked[node] = key + " " + string(raw) + asked[node], asks[node] = key+" "+string(raw), ask mu.Unlock() switch node { case "g14": @@ -98,6 +100,10 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) { if got[0].Node != "ace" || got[0].Why != "" || !strings.Contains(string(got[0].Body), "12345678901234567890") { t.Errorf("ace: %+v", got[0]) } + // Design 47 H2 §775, H-A-405: each machine is asked as a kept call under the step's ask with the node, until compose-ask-bound. + if asks["ace"] != (process.KeptAsk{ID: "step-1.ace", Bound: time.Second}) || asks["g14"].ID != "step-1.g14" { + t.Errorf("the kept asks %+v", asks) + } if !strings.Contains(got[1].Why, "no responders") || !strings.Contains(got[2].Why, "unready") || got[1].Body != nil || got[2].Body != nil { t.Errorf("g14 and novox: %+v %+v", got[1], got[2]) } @@ -115,7 +121,7 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) { func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) { input := `{"n":12345678901234567890,"big":"` + strings.Repeat("x", 1<<20) + `"}` file := base64.StdEncoding.EncodeToString([]byte(input)) - call := func(key string, body any) (json.RawMessage, error) { + call := func(_ process.KeptAsk, key string, body any) (json.RawMessage, error) { _, node, _ := strings.Cut(key, "@") answer := `{"candidate":"run-r1","node":"` + node + `","verdict":"composed","modules":[],"own":{},"took":1,"v":1,"chunks":3` if node == "g14" { @@ -155,7 +161,7 @@ func gatherAll(c *compositionInputs, run meshwants.StepRun, away func(string) bo continue } wg.Add(1) - go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key) }() + go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key, "step-1") }() } wg.Wait() return out, nil diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go index f5d3088be..daddb921d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go @@ -126,7 +126,7 @@ func (r *compositionRun) run(ctx context.Context, id string, step meshwants.Step continue } if j.started { - go r.start(ctx, j.key, j.run, prep) + go r.start(ctx, j.key, j.run, prep, id) } keys = append(keys, j.key) } @@ -241,9 +241,9 @@ func (p *runPrep) get(ctx context.Context) error { return p.err } -// start composes the machine of the run it started (§718): the node's input of this moment, its request kept once in `inputs`, +// start composes the machine of the run the step asked under stepAsk started (§718): the node's input of this moment, its request kept once in `inputs`, // the composer program chosen and recorded with the request's digest, and one run ask put. A fault ends the run could not run. -func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep) { +func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep, stepAsk string) { node, ask := run.Machine, run.Ask end := func(said string) { if _, err := r.runs.endRun(key, ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil); err != nil { @@ -269,7 +269,7 @@ func (r *compositionRun) start(ctx context.Context, key string, run meshwants.Co end(node + ": " + err.Error()) return } - in := r.inputs.of(ctx, node, run.Candidate) + in := r.inputs.of(ctx, node, run.Candidate, stepAsk) // a kept call under the starting step's ask with the node (H-A-405) if in.Why != "" { end(node + " gave no composition input: " + in.Why) return diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go index 60cc21601..2d94118fc 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "git.novox.be/novox/mesh-sdk/go/process" "maps" "slices" "strings" @@ -103,7 +104,7 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) * } return nil, errors.New("no such verb " + key) } - engines := func(key string, _ any) (json.RawMessage, error) { + engines := func(_ process.KeptAsk, key string, _ any) (json.RawMessage, error) { _, node, _ := strings.Cut(key, "@") // addressed to one node (A-237b) return json.RawMessage(`{"candidate":"main.7","node":"` + node + `","verdict":"composed","input":{"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 548b74c34..cc6e8168b 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -671,7 +671,11 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } // The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine. if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil { - run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, + askBound, err := composeAskBound() + if err != nil { + return err + } + run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFileKept, bound: askBound}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, trunk: trunkOf(iv, w.trunkName), newest: iv.newestOf(), watched: iv.watchedOf, assigned: w.assignedOn, assignment: w.assignmentOf, findings: findings, running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf} // The seat's work queue (design 47 H2 §777): pulled through its one worker, each ask taken once in the state taken. takenBound, err := askTakenBound() diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index cfed17a95..3a82d04a2 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-assigner/internal/buckets" "mesh-delivery-assigner/internal/puts" "os" @@ -295,6 +296,9 @@ func (b *fakeBus) CallOn(node, key string, body any) (json.RawMessage, error) { } func (b *fakeBus) CallKept(key string, body any) (json.RawMessage, error) { return b.Call(key, body) } func (b *fakeBus) CallFile(key string, body any) (json.RawMessage, error) { return b.Call(key, body) } +func (b *fakeBus) CallFileKept(_ process.KeptAsk, key string, body any) (json.RawMessage, error) { + return b.Call(key, body) +} func (b *fakeBus) Emit(eventType string, body any) error { b.events = append(b.events, fmt.Sprintf("%s %+v", eventType, body)) return nil diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go index b42fa8594..a3deaf66d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go @@ -21,6 +21,7 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-assigner/internal/assign" ) @@ -44,6 +45,8 @@ type caller interface { CallKept(key string, body any) (json.RawMessage, error) // CallFile is Call of a verb marked chunks, its answer read whole (process.Conn.CallFile; design 47 G A-291). CallFile(key string, body any) (json.RawMessage, error) + // CallFileKept is CallFile as a kept call carrying its ask's id, until its bound (process.Conn.CallFileKept; H2 §775). + CallFileKept(ask process.KeptAsk, key string, body any) (json.RawMessage, error) Emit(eventType string, body any) error } diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index b7bc88695..1bdb3c35d 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -172,11 +172,11 @@ func TestManifest(t *testing.T) { } file, _ := os.ReadFile("module.json") if err := json.Unmarshal(file, &raw); err != nil || raw.Config["outside-wait-minutes"].Default != float64(30) || raw.Config["outside-settle-seconds"].Default != float64(90) || - raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 13 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) || + raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 14 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) || raw.Config["composition-runs-cap-mib"].Default != float64(16) || raw.Config["composition-runs-kept"].Default != "24h" || raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) || // design 47 H2 §775, H-A-381 (2): the taken key's bound and the kept-call pace, runtime settings - raw.Config["ask-taken-bound"].Default != "10m" || raw.Config["kept-call-pace-first"].Default != "1s" || raw.Config["kept-call-pace-max"].Default != "30s" { + raw.Config["ask-taken-bound"].Default != "10m" || raw.Config["kept-call-pace-first"].Default != "1s" || raw.Config["kept-call-pace-max"].Default != "30s" || raw.Config["compose-ask-bound"].Default != "10s" { t.Errorf("config = %+v (%v), want outside-wait-minutes of 30, outside-settle-seconds of 90 and ask-operator-for of take-off,remove-node,stops (A-425a)", raw.Config, err) } for _, f := range []struct{ env, path, key string }{ diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index e1502c7d3..699f5dfd1 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -238,6 +238,13 @@ "default": "10m", "runtime": true }, + "compose-ask-bound": { + "type": "string", + "description": "how long the assigner asks a machine's node-engine compose-candidate again through a transport fault before the machine gave no input, a Go duration such as 10s (design 47 H2 §775).", + "trusted": false, + "default": "10s", + "runtime": true + }, "kept-call-pace-first": { "type": "string", "description": "the first wait before a kept call met by a transport fault is made again, doubled per call up to kept-call-pace-max, a Go duration (design 47 H2 §775).",