A1: the assigner asks each machine's compose-candidate as a kept call
A composition step ended at a node-engine's first "no responders": the engine between two lives gave no input. The holder of mesh-wants now asks compose-candidate through mesh-sdk's kept chunked call, under the ask id <step ask>.<node> the engine takes once, called again at the kept-call pace through transport faults until compose-ask-bound (a runtime setting, 10s). Builds: design 47 H2 §775; H-A-405, H-A-364
This commit is contained in:
@@ -15,6 +15,8 @@ import (
|
||||
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// The inputs of a composition run (design 47 G §354, §718, A-243, A-252): one request per candidate and machine, carrying that
|
||||
@@ -72,28 +74,44 @@ type nodeInput struct {
|
||||
// compositionInputs asks the machines of a step for their inputs.
|
||||
type compositionInputs struct {
|
||||
machines func(module, branch string) []string
|
||||
// callFile asks a seat verb marked chunks, addressed to one node as `seat:<seat>.<verb>@<node>`, and answers the verb's answer with
|
||||
// the whole file received added as its member `file`, base64 (process.Conn.CallFile; design 47 G A-237b, A-237c, A-249). The call
|
||||
// token is the body's `call`.
|
||||
callFile func(key string, body any) (json.RawMessage, error)
|
||||
bound time.Duration
|
||||
// callFile asks a seat verb marked chunks as a kept call (process.Conn.CallFileKept; design 47 H2 §775, H-A-405), addressed to
|
||||
// one node as `seat:<seat>.<verb>@<node>`, carrying the ask's id and stopping at its bound, and answers the verb's answer with
|
||||
// the whole file received added as its member `file`, base64 (design 47 G A-237b, A-237c, A-249). The call token is the
|
||||
// body's `call`.
|
||||
callFile func(ask process.KeptAsk, key string, body any) (json.RawMessage, error)
|
||||
// bound is compose-ask-bound: how long a machine's compose-candidate is asked again through a transport fault (§775).
|
||||
bound time.Duration
|
||||
}
|
||||
|
||||
// of asks the machine node for its input with the candidate's key (§718): asked only by the composition step that starts the
|
||||
// machine's composition run, once per candidate and machine. A node that is silent, unready or answers no input has Why set.
|
||||
func (c *compositionInputs) of(ctx context.Context, node, candidate string) nodeInput {
|
||||
// composeAskBoundSetting is the holder's bound on its compose-candidate asks (design 47 H2 §775): past it the machine gave no
|
||||
// input, naming every call.
|
||||
const (
|
||||
composeAskBoundSetting = "compose-ask-bound"
|
||||
defaultComposeAskBound = 10 * time.Second
|
||||
)
|
||||
|
||||
// composeAskBound reads the setting compose-ask-bound (§781).
|
||||
func composeAskBound() (time.Duration, error) {
|
||||
d, err := stdio.SettingDuration(composeAskBoundSetting, defaultComposeAskBound)
|
||||
if err != nil {
|
||||
return 0, errors.New("the assigner's " + err.Error())
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// composeAskID is the id of the compose-candidate ask of the step asked under stepAsk to node (H-A-405 (2)): one per node per
|
||||
// step, so the engines' mesh-wide state taken keeps each apart, and a repeat to the same node carries the same id.
|
||||
func composeAskID(stepAsk, node string) string { return stepAsk + "." + node }
|
||||
|
||||
// of asks the machine node for its input with the candidate's key (§718), as a kept call under the id of stepAsk, the step
|
||||
// that starts the machine's composition run: asked once per candidate and machine, again only through a transport fault until
|
||||
// compose-ask-bound (§775). A node that is silent past the bound, unready or answers no input has Why set.
|
||||
func (c *compositionInputs) of(ctx context.Context, node, candidate, stepAsk string) nodeInput {
|
||||
bound := c.bound
|
||||
if bound <= 0 {
|
||||
bound = nec.AnswerBound
|
||||
bound = defaultComposeAskBound
|
||||
}
|
||||
return c.one(ctx, bound, node, candidate)
|
||||
}
|
||||
|
||||
// one asks one node for its input.
|
||||
func (c *compositionInputs) one(ctx context.Context, bound time.Duration, node, key string) nodeInput {
|
||||
in := nodeInput{Node: node}
|
||||
at, cancel := context.WithTimeout(ctx, bound)
|
||||
defer cancel()
|
||||
type reply struct {
|
||||
raw json.RawMessage
|
||||
err error
|
||||
@@ -106,13 +124,14 @@ func (c *compositionInputs) one(ctx context.Context, bound time.Duration, node,
|
||||
got <- reply{nil, err}
|
||||
return
|
||||
}
|
||||
raw, err := c.callFile(composeCandidateKey+"@"+node, ne.ComposeCandidateInput{Candidate: key, Input: true, Call: hex.EncodeToString(bits[:])})
|
||||
raw, err := c.callFile(process.KeptAsk{ID: composeAskID(stepAsk, node), Bound: bound}, composeCandidateKey+"@"+node,
|
||||
ne.ComposeCandidateInput{Candidate: candidate, Input: true, Call: hex.EncodeToString(bits[:])})
|
||||
got <- reply{raw, err}
|
||||
}()
|
||||
var raw json.RawMessage
|
||||
select {
|
||||
case <-at.Done():
|
||||
in.Why = "no answer within its bound: " + at.Err().Error()
|
||||
case <-ctx.Done():
|
||||
in.Why = "its step's wait ended: " + context.Cause(ctx).Error()
|
||||
return in
|
||||
case r := <-got:
|
||||
if r.err != nil {
|
||||
|
||||
+10
-4
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -71,11 +72,12 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
|
||||
// the machines are asked side by side, so the record of what each was asked is guarded.
|
||||
var mu sync.Mutex
|
||||
asked := map[string]string{}
|
||||
call := func(key string, body any) (json.RawMessage, error) {
|
||||
asks := map[string]process.KeptAsk{}
|
||||
call := func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
key, node, _ := strings.Cut(key, "@") // addressed to one node (A-237b)
|
||||
mu.Lock()
|
||||
asked[node] = key + " " + string(raw)
|
||||
asked[node], asks[node] = key+" "+string(raw), ask
|
||||
mu.Unlock()
|
||||
switch node {
|
||||
case "g14":
|
||||
@@ -98,6 +100,10 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
|
||||
if got[0].Node != "ace" || got[0].Why != "" || !strings.Contains(string(got[0].Body), "12345678901234567890") {
|
||||
t.Errorf("ace: %+v", got[0])
|
||||
}
|
||||
// Design 47 H2 §775, H-A-405: each machine is asked as a kept call under the step's ask with the node, until compose-ask-bound.
|
||||
if asks["ace"] != (process.KeptAsk{ID: "step-1.ace", Bound: time.Second}) || asks["g14"].ID != "step-1.g14" {
|
||||
t.Errorf("the kept asks %+v", asks)
|
||||
}
|
||||
if !strings.Contains(got[1].Why, "no responders") || !strings.Contains(got[2].Why, "unready") || got[1].Body != nil || got[2].Body != nil {
|
||||
t.Errorf("g14 and novox: %+v %+v", got[1], got[2])
|
||||
}
|
||||
@@ -115,7 +121,7 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
|
||||
func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) {
|
||||
input := `{"n":12345678901234567890,"big":"` + strings.Repeat("x", 1<<20) + `"}`
|
||||
file := base64.StdEncoding.EncodeToString([]byte(input))
|
||||
call := func(key string, body any) (json.RawMessage, error) {
|
||||
call := func(_ process.KeptAsk, key string, body any) (json.RawMessage, error) {
|
||||
_, node, _ := strings.Cut(key, "@")
|
||||
answer := `{"candidate":"run-r1","node":"` + node + `","verdict":"composed","modules":[],"own":{},"took":1,"v":1,"chunks":3`
|
||||
if node == "g14" {
|
||||
@@ -155,7 +161,7 @@ func gatherAll(c *compositionInputs, run meshwants.StepRun, away func(string) bo
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key) }()
|
||||
go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key, "step-1") }()
|
||||
}
|
||||
wg.Wait()
|
||||
return out, nil
|
||||
|
||||
@@ -126,7 +126,7 @@ func (r *compositionRun) run(ctx context.Context, id string, step meshwants.Step
|
||||
continue
|
||||
}
|
||||
if j.started {
|
||||
go r.start(ctx, j.key, j.run, prep)
|
||||
go r.start(ctx, j.key, j.run, prep, id)
|
||||
}
|
||||
keys = append(keys, j.key)
|
||||
}
|
||||
@@ -241,9 +241,9 @@ func (p *runPrep) get(ctx context.Context) error {
|
||||
return p.err
|
||||
}
|
||||
|
||||
// start composes the machine of the run it started (§718): the node's input of this moment, its request kept once in `inputs`,
|
||||
// start composes the machine of the run the step asked under stepAsk started (§718): the node's input of this moment, its request kept once in `inputs`,
|
||||
// the composer program chosen and recorded with the request's digest, and one run ask put. A fault ends the run could not run.
|
||||
func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep) {
|
||||
func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep, stepAsk string) {
|
||||
node, ask := run.Machine, run.Ask
|
||||
end := func(said string) {
|
||||
if _, err := r.runs.endRun(key, ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil); err != nil {
|
||||
@@ -269,7 +269,7 @@ func (r *compositionRun) start(ctx context.Context, key string, run meshwants.Co
|
||||
end(node + ": " + err.Error())
|
||||
return
|
||||
}
|
||||
in := r.inputs.of(ctx, node, run.Candidate)
|
||||
in := r.inputs.of(ctx, node, run.Candidate, stepAsk) // a kept call under the starting step's ask with the node (H-A-405)
|
||||
if in.Why != "" {
|
||||
end(node + " gave no composition input: " + in.Why)
|
||||
return
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -103,7 +104,7 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
|
||||
}
|
||||
return nil, errors.New("no such verb " + key)
|
||||
}
|
||||
engines := func(key string, _ any) (json.RawMessage, error) {
|
||||
engines := func(_ process.KeptAsk, key string, _ any) (json.RawMessage, error) {
|
||||
_, node, _ := strings.Cut(key, "@") // addressed to one node (A-237b)
|
||||
return json.RawMessage(`{"candidate":"main.7","node":"` + node + `","verdict":"composed","input":{"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
|
||||
}
|
||||
|
||||
@@ -671,7 +671,11 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
// The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine.
|
||||
if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil {
|
||||
run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept,
|
||||
askBound, err := composeAskBound()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFileKept, bound: askBound}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept,
|
||||
trunk: trunkOf(iv, w.trunkName), newest: iv.newestOf(), watched: iv.watchedOf, assigned: w.assignedOn, assignment: w.assignmentOf, findings: findings, running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf}
|
||||
// The seat's work queue (design 47 H2 §777): pulled through its one worker, each ask taken once in the state taken.
|
||||
takenBound, err := askTakenBound()
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"mesh-delivery-assigner/internal/puts"
|
||||
"os"
|
||||
@@ -295,6 +296,9 @@ func (b *fakeBus) CallOn(node, key string, body any) (json.RawMessage, error) {
|
||||
}
|
||||
func (b *fakeBus) CallKept(key string, body any) (json.RawMessage, error) { return b.Call(key, body) }
|
||||
func (b *fakeBus) CallFile(key string, body any) (json.RawMessage, error) { return b.Call(key, body) }
|
||||
func (b *fakeBus) CallFileKept(_ process.KeptAsk, key string, body any) (json.RawMessage, error) {
|
||||
return b.Call(key, body)
|
||||
}
|
||||
func (b *fakeBus) Emit(eventType string, body any) error {
|
||||
b.events = append(b.events, fmt.Sprintf("%s %+v", eventType, body))
|
||||
return nil
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -44,6 +45,8 @@ type caller interface {
|
||||
CallKept(key string, body any) (json.RawMessage, error)
|
||||
// CallFile is Call of a verb marked chunks, its answer read whole (process.Conn.CallFile; design 47 G A-291).
|
||||
CallFile(key string, body any) (json.RawMessage, error)
|
||||
// CallFileKept is CallFile as a kept call carrying its ask's id, until its bound (process.Conn.CallFileKept; H2 §775).
|
||||
CallFileKept(ask process.KeptAsk, key string, body any) (json.RawMessage, error)
|
||||
Emit(eventType string, body any) error
|
||||
}
|
||||
|
||||
|
||||
@@ -172,11 +172,11 @@ func TestManifest(t *testing.T) {
|
||||
}
|
||||
file, _ := os.ReadFile("module.json")
|
||||
if err := json.Unmarshal(file, &raw); err != nil || raw.Config["outside-wait-minutes"].Default != float64(30) || raw.Config["outside-settle-seconds"].Default != float64(90) ||
|
||||
raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 13 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) ||
|
||||
raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 14 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) ||
|
||||
raw.Config["composition-runs-cap-mib"].Default != float64(16) || raw.Config["composition-runs-kept"].Default != "24h" ||
|
||||
raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) ||
|
||||
// design 47 H2 §775, H-A-381 (2): the taken key's bound and the kept-call pace, runtime settings
|
||||
raw.Config["ask-taken-bound"].Default != "10m" || raw.Config["kept-call-pace-first"].Default != "1s" || raw.Config["kept-call-pace-max"].Default != "30s" {
|
||||
raw.Config["ask-taken-bound"].Default != "10m" || raw.Config["kept-call-pace-first"].Default != "1s" || raw.Config["kept-call-pace-max"].Default != "30s" || raw.Config["compose-ask-bound"].Default != "10s" {
|
||||
t.Errorf("config = %+v (%v), want outside-wait-minutes of 30, outside-settle-seconds of 90 and ask-operator-for of take-off,remove-node,stops (A-425a)", raw.Config, err)
|
||||
}
|
||||
for _, f := range []struct{ env, path, key string }{
|
||||
|
||||
@@ -238,6 +238,13 @@
|
||||
"default": "10m",
|
||||
"runtime": true
|
||||
},
|
||||
"compose-ask-bound": {
|
||||
"type": "string",
|
||||
"description": "how long the assigner asks a machine's node-engine compose-candidate again through a transport fault before the machine gave no input, a Go duration such as 10s (design 47 H2 §775).",
|
||||
"trusted": false,
|
||||
"default": "10s",
|
||||
"runtime": true
|
||||
},
|
||||
"kept-call-pace-first": {
|
||||
"type": "string",
|
||||
"description": "the first wait before a kept call met by a transport fault is made again, doubled per call up to kept-call-pace-max, a Go duration (design 47 H2 §775).",
|
||||
|
||||
Reference in New Issue
Block a user