diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks_tools.go new file mode 100644 index 000000000..251610e5e --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks_tools.go @@ -0,0 +1,162 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// The seat mesh-wants' work queue read and steered by hand (design 47 H2 §777, the tools table; H-A-410): assigner_asks +// answers what waits on the seat's queue (read by the runtime without taking it) and what the assigner took and keeps +// dropped in its state taken; assigner_ask_drop ends one ask through the seat's act drop, written in the hand-act log first. + +const ( + asksTool = "assigner_asks" + askDropTool = "assigner_ask_drop" +) + +// asksAnswerMost is the most an asks answer carries of the taken keys (§811: at most 768 KB with capped and total). +const asksAnswerMost = 768 << 10 + +// takenReader is what assigner_asks reads of the state taken (stdio.KeptState). +type takenReader interface { + Keys() ([]string, error) + Get(key string) (*stdio.StateEntry, error) +} + +// asksOf is assigner_asks' answer: the asks on the seat's queue, each waiting, waiting again or held, the ids taken (key mesh-wants.) and the drops of +// asks not pulled yet (dropped.), each list cut at the answer's bound with capped and total. +func asksOf(waiting func(seat string) (stdio.Waiting, error), taken takenReader) (map[string]any, error) { + w, err := waiting(meshwants.Seat) + if err != nil { + return nil, fmt.Errorf("the asks waiting on the seat %s's work queue are not read: %w", meshwants.Seat, err) + } + keys, err := taken.Keys() + if err != nil { + return nil, fmt.Errorf("the state %s is not listed: %w", process.TakenState, err) + } + slices.Sort(keys) + // An ask the queue handed out and not acknowledged (again) is held when this holder took it, else it waits again after a + // nak (H-A-410's addenda): the bus says what it knows, the state taken the rest. + size := 0 + waits := make([]map[string]any, 0, len(w.Asks)) + for _, a := range w.Asks { + size += len(a.ID) + len(a.Module) + 64 + state := "waiting" + if a.Again { + state = "waiting again" + if slices.Contains(keys, process.TakenKey(meshwants.Seat, a.ID)) { + state = "held" + } + } + waits = append(waits, map[string]any{"id": a.ID, "module": a.Module, "age_seconds": a.AgeSeconds, "state": state}) + } + takenIDs, dropped := []string{}, []map[string]any{} + total, capped := 0, w.Capped + for _, k := range keys { + if id, ok := strings.CutPrefix(k, process.TakenKey(meshwants.Seat, "")); ok { + total++ + if size += len(id) + 4; size > asksAnswerMost { + capped = true + continue + } + takenIDs = append(takenIDs, id) + continue + } + id, ok := strings.CutPrefix(k, droppedPrefix) + if !ok { + continue + } + total++ + e, err := taken.Get(k) + if err != nil || e == nil { + continue + } + var d askDrop + if json.Unmarshal(e.Value, &d) != nil { + continue + } + if size += len(id) + len(d.By) + len(d.Why) + 64; size > asksAnswerMost { + capped = true + continue + } + dropped = append(dropped, map[string]any{"id": id, "by": d.By, "why": d.Why, "at": d.At}) + } + return map[string]any{"waiting": waits, "taken": takenIDs, "dropped": dropped, "capped": capped, "total": total + len(w.Asks)}, nil +} + +// asksTools are the module's tools of the seat's work queue. +func asksTools(waiting func(string) (stdio.Waiting, error), taken takenReader, ask func(string, any) (json.RawMessage, error)) []stdio.Tool { + return []stdio.Tool{ + {Name: asksTool, Description: "The asks of the seat mesh-wants' work queue (design 47 H2 §777): each one on it (its id, the module " + + "that asked it, its age and whether it waits, waits again after a hand-back or is held here), the ids the assigner took (kept for ask-taken-bound) and the drops of asks not pulled yet, " + + "cut at 768 KB with capped and total. Read-only.", + Input: schema(map[string]string{}, nil), + Run: func(map[string]any) (any, error) { + return asksOf(waiting, taken) + }}, + {Name: askDropTool, Description: "End one ask of the seat mesh-wants by hand (design 47 H2 §777, §778): written in the hand-act log " + + "first, then the seat's act drop ends it: one running is cancelled, one waiting is ended when it is pulled; either ends " + + "its step could not run naming the drop and who asked it.", + Input: schema(map[string]string{"ask": "the ask's id", "why": "why the ask is ended"}, []string{"ask", "why"}), + Run: func(args map[string]any) (any, error) { + id, why := strings.TrimSpace(arg(args, "ask")), strings.TrimSpace(arg(args, "why")) + if id == "" || why == "" { + return nil, errors.New("name the ask and why it is ended") + } + by := stdio.CallerOf(args) + taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: askDropTool + " " + id, Why: why + " (" + by + ")", Cause: "ask-drop"}) + if err != nil { + return nil, fmt.Errorf("the hand act was not recorded, so nothing was dropped: %w", err) + } + raw, err := ask("seat:"+meshwants.Seat+"."+meshwants.VerbDrop, meshwants.DropInput{ID: id, By: askDropTool + " (" + by + ")", Why: why}) + if err != nil { + return nil, err + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("%s answered what is not JSON: %w", meshwants.VerbDrop, err) + } + out["hand-act"] = taken.Log + return out, nil + }}, + } +} + +// keepNowTool is the tool assigner_keep_now (design 47 H2 §813 (2); H-A-417): written in the hand-act log first, then the seat +// mesh-wants' act keep-now runs the keeping of the assignment graph's revisions at once in the assigner's process. +const keepNowTool = "assigner_keep_now" + +func keepNowTools(ask func(string, any) (json.RawMessage, error)) []stdio.Tool { + return []stdio.Tool{{Name: keepNowTool, Description: "Keep the assignment graph's revisions now (design 47 H2 §813 (2)): written in the " + + "hand-act log first, then the assigner keeps the pass's newest computed revision at once and answers the revision kept, or " + + "the fault the keeping met (a failed keeping runs again after keeping-retry; keeping-failed is raised at the third in a row).", + Input: schema(map[string]string{"why": "why the keeping runs now"}, []string{"why"}), + Run: func(args map[string]any) (any, error) { + why := strings.TrimSpace(arg(args, "why")) + if why == "" { + return nil, errors.New("say why the keeping runs now") + } + taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: keepNowTool, Why: why + " (" + stdio.CallerOf(args) + ")", Cause: "keep-now"}) + if err != nil { + return nil, fmt.Errorf("the hand act was not recorded, so nothing was kept: %w", err) + } + raw, err := ask("seat:"+meshwants.Seat+"."+meshwants.VerbKeepNow, meshwants.KeepNowInput{Why: why}) + if err != nil { + return nil, err + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("%s answered what is not JSON: %w", meshwants.VerbKeepNow, err) + } + out["hand-act"] = taken.Log + return out, nil + }}} +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go index 52c522630..afb8b1fb2 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go @@ -15,6 +15,8 @@ import ( nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" ) // The inputs of a composition run (design 47 G §354, §718, A-243, A-252): one request per candidate and machine, carrying that @@ -72,28 +74,44 @@ type nodeInput struct { // compositionInputs asks the machines of a step for their inputs. type compositionInputs struct { machines func(module, branch string) []string - // callFile asks a seat verb marked chunks, addressed to one node as `seat:.@`, and answers the verb's answer with - // the whole file received added as its member `file`, base64 (process.Conn.CallFile; design 47 G A-237b, A-237c, A-249). The call - // token is the body's `call`. - callFile func(key string, body any) (json.RawMessage, error) - bound time.Duration + // callFile asks a seat verb marked chunks as a kept call (process.Conn.CallFileKept; design 47 H2 §775, H-A-405), addressed to + // one node as `seat:.@`, carrying the ask's id and stopping at its bound, and answers the verb's answer with + // the whole file received added as its member `file`, base64 (design 47 G A-237b, A-237c, A-249). The call token is the + // body's `call`. + callFile func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) + // bound is compose-ask-bound: how long a machine's compose-candidate is asked again through a transport fault (§775). + bound time.Duration } -// of asks the machine node for its input with the candidate's key (§718): asked only by the composition step that starts the -// machine's composition run, once per candidate and machine. A node that is silent, unready or answers no input has Why set. -func (c *compositionInputs) of(ctx context.Context, node, candidate string) nodeInput { +// composeAskBoundSetting is the holder's bound on its compose-candidate asks (design 47 H2 §775): past it the machine gave no +// input, naming every call. +const ( + composeAskBoundSetting = "compose-ask-bound" + defaultComposeAskBound = 10 * time.Second +) + +// composeAskBound reads the setting compose-ask-bound (§781). +func composeAskBound() (time.Duration, error) { + d, err := stdio.SettingDuration(composeAskBoundSetting, defaultComposeAskBound) + if err != nil { + return 0, errors.New("the assigner's " + err.Error()) + } + return d, nil +} + +// composeAskID is the id of the compose-candidate ask of the step asked under stepAsk to node (H-A-405 (2)): one per node per +// step, so the engines' mesh-wide state taken keeps each apart, and a repeat to the same node carries the same id. +func composeAskID(stepAsk, node string) string { return stepAsk + "." + node } + +// of asks the machine node for its input with the candidate's key (§718), as a kept call under the id of stepAsk, the step +// that starts the machine's composition run: asked once per candidate and machine, again only through a transport fault until +// compose-ask-bound (§775). A node that is silent past the bound, unready or answers no input has Why set. +func (c *compositionInputs) of(ctx context.Context, node, candidate, stepAsk string) nodeInput { bound := c.bound if bound <= 0 { - bound = nec.AnswerBound + bound = defaultComposeAskBound } - return c.one(ctx, bound, node, candidate) -} - -// one asks one node for its input. -func (c *compositionInputs) one(ctx context.Context, bound time.Duration, node, key string) nodeInput { in := nodeInput{Node: node} - at, cancel := context.WithTimeout(ctx, bound) - defer cancel() type reply struct { raw json.RawMessage err error @@ -106,13 +124,14 @@ func (c *compositionInputs) one(ctx context.Context, bound time.Duration, node, got <- reply{nil, err} return } - raw, err := c.callFile(composeCandidateKey+"@"+node, ne.ComposeCandidateInput{Candidate: key, Input: true, Call: hex.EncodeToString(bits[:])}) + raw, err := c.callFile(process.KeptAsk{ID: composeAskID(stepAsk, node), Bound: bound}, composeCandidateKey+"@"+node, + ne.ComposeCandidateInput{Candidate: candidate, Input: true, Call: hex.EncodeToString(bits[:])}) got <- reply{raw, err} }() var raw json.RawMessage select { - case <-at.Done(): - in.Why = "no answer within its bound: " + at.Err().Error() + case <-ctx.Done(): + in.Why = "its step's wait ended: " + context.Cause(ctx).Error() return in case r := <-got: if r.err != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go index d403de527..596bcbab5 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "errors" + "git.novox.be/novox/mesh-sdk/go/process" "regexp" "strings" "sync" @@ -71,11 +72,12 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) { // the machines are asked side by side, so the record of what each was asked is guarded. var mu sync.Mutex asked := map[string]string{} - call := func(key string, body any) (json.RawMessage, error) { + asks := map[string]process.KeptAsk{} + call := func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) { raw, _ := json.Marshal(body) key, node, _ := strings.Cut(key, "@") // addressed to one node (A-237b) mu.Lock() - asked[node] = key + " " + string(raw) + asked[node], asks[node] = key+" "+string(raw), ask mu.Unlock() switch node { case "g14": @@ -98,6 +100,10 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) { if got[0].Node != "ace" || got[0].Why != "" || !strings.Contains(string(got[0].Body), "12345678901234567890") { t.Errorf("ace: %+v", got[0]) } + // Design 47 H2 §775, H-A-405: each machine is asked as a kept call under the step's ask with the node, until compose-ask-bound. + if asks["ace"] != (process.KeptAsk{ID: "step-1.ace", Bound: time.Second}) || asks["g14"].ID != "step-1.g14" { + t.Errorf("the kept asks %+v", asks) + } if !strings.Contains(got[1].Why, "no responders") || !strings.Contains(got[2].Why, "unready") || got[1].Body != nil || got[2].Body != nil { t.Errorf("g14 and novox: %+v %+v", got[1], got[2]) } @@ -115,7 +121,7 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) { func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) { input := `{"n":12345678901234567890,"big":"` + strings.Repeat("x", 1<<20) + `"}` file := base64.StdEncoding.EncodeToString([]byte(input)) - call := func(key string, body any) (json.RawMessage, error) { + call := func(_ process.KeptAsk, key string, body any) (json.RawMessage, error) { _, node, _ := strings.Cut(key, "@") answer := `{"candidate":"run-r1","node":"` + node + `","verdict":"composed","modules":[],"own":{},"took":1,"v":1,"chunks":3` if node == "g14" { @@ -155,7 +161,7 @@ func gatherAll(c *compositionInputs, run meshwants.StepRun, away func(string) bo continue } wg.Add(1) - go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key) }() + go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key, "step-1") }() } wg.Wait() return out, nil diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go index f5d3088be..2188cd04a 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go @@ -22,6 +22,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-assigner/internal/assign" ) @@ -55,6 +56,9 @@ type compositionRun struct { keeper *inputKeeper // call asks a seat verb held once for the mesh (process.Conn.CallKept). call func(key string, body any) (json.RawMessage, error) + // callFile asks a seat verb held once for the mesh whose answer comes in chunks, as a kept call (process.Conn.CallFileKept), + // answering the verb's answer with the bytes received as its member `file`, base64: the validate step's seat-list (H-A-443). + callFile func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) // trunk is the trunk's catalog half as the assigner holds it. trunk func() (mcc.Half, bool) // newest is the newest catalog half the assigner's views hold of each watched branch (halves.views), and watched a module's @@ -126,7 +130,7 @@ func (r *compositionRun) run(ctx context.Context, id string, step meshwants.Step continue } if j.started { - go r.start(ctx, j.key, j.run, prep) + go r.start(ctx, j.key, j.run, prep, id) } keys = append(keys, j.key) } @@ -241,9 +245,9 @@ func (p *runPrep) get(ctx context.Context) error { return p.err } -// start composes the machine of the run it started (§718): the node's input of this moment, its request kept once in `inputs`, +// start composes the machine of the run the step asked under stepAsk started (§718): the node's input of this moment, its request kept once in `inputs`, // the composer program chosen and recorded with the request's digest, and one run ask put. A fault ends the run could not run. -func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep) { +func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep, stepAsk string) { node, ask := run.Machine, run.Ask end := func(said string) { if _, err := r.runs.endRun(key, ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil); err != nil { @@ -269,7 +273,7 @@ func (r *compositionRun) start(ctx context.Context, key string, run meshwants.Co end(node + ": " + err.Error()) return } - in := r.inputs.of(ctx, node, run.Candidate) + in := r.inputs.of(ctx, node, run.Candidate, stepAsk) // a kept call under the starting step's ask with the node (H-A-405) if in.Why != "" { end(node + " gave no composition input: " + in.Why) return diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go index 76646e847..2d94118fc 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "git.novox.be/novox/mesh-sdk/go/process" "maps" "slices" "strings" @@ -103,7 +104,7 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) * } return nil, errors.New("no such verb " + key) } - engines := func(key string, _ any) (json.RawMessage, error) { + engines := func(_ process.KeptAsk, key string, _ any) (json.RawMessage, error) { _, node, _ := strings.Cut(key, "@") // addressed to one node (A-237b) return json.RawMessage(`{"candidate":"main.7","node":"` + node + `","verdict":"composed","input":{"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil } @@ -638,7 +639,7 @@ func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) { } got := map[string]meshwants.StepEnded{} for _, m := range []string{"a", "b", "d"} { - steps.take(context.Background(), "ask-"+m, step(m, false)) + pullOne(steps, "ask-"+m, step(m, false)) e := <-ends got[e.Module] = e } @@ -649,7 +650,7 @@ func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) { mu.Lock() fixed = true mu.Unlock() - steps.take(context.Background(), "ask-a2", step("a", true)) + pullOne(steps, "ask-a2", step("a", true)) moved := map[string]meshwants.StepEnded{} for range 2 { select { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go index 7f6e6acef..fc282ed30 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go @@ -126,7 +126,7 @@ func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) { from := b.journal.mark() _, stop := b.startStoppable(t) - b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second) + b.journal.await(t, from, "the composition step is pulled from", "the composition step served", 30*time.Second) select { case <-b.seatListRead: case <-time.After(30 * time.Second): diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go index 587861683..6bf171b15 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go @@ -2,14 +2,11 @@ package main import ( "context" - "encoding/json" + "errors" "sync" "time" - "github.com/nats-io/nats.go" - meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" - "git.novox.be/novox/mesh-sdk/go/process" ) // The composition step (design 47 G §22, §354, §63): the assigner takes the seat mesh-wants' accepted kind `step`, the @@ -44,9 +41,18 @@ type compositionSteps struct { logf func(string, ...any) // ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own. ticks func(d time.Duration) (<-chan time.Time, func()) - mu sync.Mutex - // stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave). - stops map[string]context.CancelFunc + // taken is the state taken (stepqueue.go): each ask pulled is taken once by its id, for takenBound; the drops of asks not + // pulled yet are kept there too (H-A-404). + taken takenStore + takenBound time.Duration + // inProgress is how often a running step's pulled ask is said in progress: the seat definition's in-progress (§777). + inProgress func() time.Duration + mu sync.Mutex + // stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave), or the ask is + // dropped (the cause an *askDrop). + stops map[string]context.CancelCauseFunc + // held are the messages pulled from the seat's work queue for each running step's ask, acknowledged once its end is said. + held map[string][]queueMsg } // progressEvery is how often a running composition step says its progress (design 47 G §496, A-262). @@ -92,20 +98,6 @@ func (p *stepPhase) get() string { return p.phase } -// take takes one ask of the accepted kind `step` by its id and body: a step that is no composition or validate step, or an -// ask with no id, is left with a log line; one taken is run in its own goroutine and its end said. -func (c *compositionSteps) take(ctx context.Context, id string, data []byte) bool { - var run meshwants.StepRun - err := json.Unmarshal(data, &run) - how := c.howOf(run.Step, false) - if err != nil || id == "" || how == nil { - c.logf("a step ask that is no composition or validate step with an id is left: %v", err) - return false - } - go c.say1(ctx, id, run, how, "") - return true -} - // howOf is what runs a step, or judges it once taken up at the start: by its step, composition or validate; nil for another. func (c *compositionSteps) howOf(step string, resumed bool) func(context.Context, string, meshwants.StepRun) stepOutcome { switch { @@ -124,16 +116,16 @@ func (c *compositionSteps) howOf(step string, resumed bool) func(context.Context // say1 runs one step with how and says its end, naming in `shared` the runs it read (§723); again is, for an end said again, the // ask of the run whose end makes it say again (sayEnd). A step whose wait was ended by its pipeline's supersede or cancel (leave) // says nothing: the runner ended it already. -func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome, again string) { +func (c *compositionSteps) say1(parent context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome, again string) { started := c.now().UTC() - ctx, stopWait := context.WithCancel(ctx) + ctx, stopWait := context.WithCancelCause(parent) c.mu.Lock() if c.stops == nil { - c.stops = map[string]context.CancelFunc{} + c.stops = map[string]context.CancelCauseFunc{} } if c.stops[id] != nil { // the step asked under id is running already: it says its own end c.mu.Unlock() - stopWait() + stopWait(nil) return } c.stops[id] = stopWait @@ -142,14 +134,25 @@ func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.St c.mu.Lock() delete(c.stops, id) c.mu.Unlock() - stopWait() + stopWait(nil) }() phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch} stop := c.sayProgress(id, run.Asker, phase) + stopInProgress := c.inProgressEvery(id) o := how(context.WithValue(ctx, phaseKey{}, phase), id, run) + stopInProgress() stop() if ctx.Err() != nil { + if parent.Err() != nil { // the assigner stops: its pulled asks are handed again to its next start + return + } + var d *askDrop + if errors.As(context.Cause(ctx), &d) { // dropped through the seat's drop (§778, H-A-401) + c.sayEnd(id, run, &started, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: d.Error(), Shared: o.Shared}, again) + return + } c.logf("the composition step %s of %s says no end: its wait ended (%v)", id, run.Pipeline, context.Cause(ctx)) + c.settle(id, true) // the runner ended it already: its ask is over return } c.sayEnd(id, run, &started, o, again) @@ -175,9 +178,13 @@ func (c *compositionSteps) sayEnd(id string, run meshwants.StepRun, started *tim msgID = id + "." + again } if err := c.say(meshwants.EventStepEnded+"."+c.node, end, msgID); err != nil { - c.logf("saying the end of the composition step %s: %v", id, err) + c.logf("saying the end of the composition step %s: %v; its ask is handed again", id, err) + c.settle(id, false) return } + c.settle(id, true) // the ask's end is said: its pulled messages leave the seat's work queue (§777) + // One journal line per ask answered, in the one place it is answered (design 47 H2 §792). + c.logf("answered %s %s %s %s %s", id, run.Pipeline, run.Module, step, o.Outcome) if c.ended != nil { c.ended(id, run, ended) } @@ -210,7 +217,7 @@ func (c *compositionSteps) leave(ids []string) { defer c.mu.Unlock() for _, id := range ids { if stop := c.stops[id]; stop != nil { - stop() + stop(nil) } } } @@ -243,21 +250,3 @@ func (c *compositionSteps) sayProgress(id, asker string, phase *stepPhase) (stop }() return func() { close(done); <-ended } } - -// stepAskID is an ask's id as the asker names it: its event id, else its message id. -func stepAskID(m *nats.Msg) string { - if id := m.Header.Get("x-event-id"); id != "" { - return id - } - return m.Header.Get(nats.MsgIdHdr) -} - -// serveCompositionSteps hears the composition step asks on the seat mesh-wants' accept subject. -func serveCompositionSteps(ctx context.Context, conn process.Subscriber, c *compositionSteps) error { - sub, err := conn.Subscribe(meshwants.AcceptSubjectStep, func(m *nats.Msg) { c.take(ctx, stepAskID(m), m.Data) }) - if err != nil { - return err - } - go func() { <-ctx.Done(); _ = sub.Drain() }() - return nil -} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go index 42615f58a..51528de73 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go @@ -106,7 +106,7 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T) from := b.journal.mark() b.start(t) - b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second) + b.journal.await(t, from, "the composition step is pulled from", "the composition step served", 30*time.Second) b.journal.await(t, from, "n3 is away (machine-away)", "the machine-away of n3 heard", 30*time.Second) select { @@ -114,14 +114,8 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T) case <-time.After(30 * time.Second): t.Fatal("the assigner did not read the seat list within 30 s") } - step, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.3.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: trunkBranch, + putStep(b.ctx, t, b.js, "step-ask-1", meshwants.StepRun{Pipeline: "main.3.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: trunkBranch, Revision: 3, Commit: engineCommit, Fingerprint: "src2:m1"}) - msg := nats.NewMsg(meshwants.AcceptSubjectStep) - msg.Header.Set("x-event-id", "step-ask-1") - msg.Data = step - if err := nc.PublishMsg(msg); err != nil { - t.Fatal(err) - } // The stand-in Go build seat takes the run asks put under the assigner's name, reads each request by the verb input and says // its end with the composer's verdict: n1 composes, n2 does not. @@ -201,6 +195,7 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T) t.Fatalf("the inputs stay kept past their step-ended: %v", keys) } } + queueEmpty(b.ctx, t, b.js) // the step's ask acknowledged at its end (design 47 H2 §777) } // readInput reads a run's input as a build seat does: the seat mesh-wants' verb input answers its digest and chunk count, then its @@ -288,9 +283,37 @@ func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T } t.Cleanup(func() { _ = ends.Unsubscribe() }) subscribed(t, nc) + // Design 47 H2 §777: a validate ask put while the assigner is stopped waits on the seat's work queue, and is taken (ask-taken) + // and ended after its start. + taken := make(chan *nats.Msg, 8) + takens, err := nc.ChanSubscribe(meshwants.SubjectAskTakenAll, taken) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = takens.Unsubscribe() }) + subscribed(t, nc) + putStep(b.ctx, t, b.js, "validate-9", meshwants.StepRun{Pipeline: trunkBranch + ".9", Step: meshwants.StepRunStepValidate, Module: "a", + Branch: trunkBranch, Revision: 9, Commit: commit}) from := b.journal.mark() b.start(t) - b.journal.await(t, from, "the composition step is heard on", "the steps served", 30*time.Second) + b.journal.await(t, from, "the composition step is pulled from", "the steps served", 30*time.Second) + select { + case m := <-taken: + var at meshwants.AskTakenEvent + if err := json.Unmarshal(m.Data, &at); err != nil || at.Ask != "validate-9" || at.Node != "n1" || at.At.IsZero() || m.Subject != meshwants.SubjectAskTaken("validate-9") { + t.Fatalf("ask-taken on %s = %+v, %v; want validate-9 taken on n1", m.Subject, at, err) + } + case <-time.After(30 * time.Second): + t.Fatal("no ask-taken within 30 s of the start for the ask put while the assigner was stopped") + } + select { + case e := <-ended: + if e.ID != "validate-9" || e.Outcome != meshwants.StepEndedOutcomeFailed || e.Said != reasonCycle+": a:x → b:y → a:x" { + t.Errorf("the validate step of a take-in making a cycle ended %+v", e) + } + case <-time.After(30 * time.Second): + t.Fatal("the validate ask put while the assigner was stopped said no end") + } select { case <-b.seatListRead: case <-time.After(30 * time.Second): @@ -306,12 +329,9 @@ func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T id := fmt.Sprintf("validate-%d", revision) step, _ := json.Marshal(meshwants.StepRun{Pipeline: fmt.Sprintf("%s.%d", trunkBranch, revision), Step: meshwants.StepRunStepValidate, Module: module, Branch: trunkBranch, Revision: revision, Commit: commit}) - msg := nats.NewMsg(meshwants.AcceptSubjectStep) - msg.Header.Set("x-event-id", id) - msg.Data = step - if err := nc.PublishMsg(msg); err != nil { - t.Fatal(err) - } + var asked meshwants.StepRun + _ = json.Unmarshal(step, &asked) + putStep(b.ctx, t, b.js, id, asked) if run != nil { run() } @@ -342,6 +362,12 @@ func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T if err != nil || len(req.Cases) != 2 || req.Cases[0].Name != changeCase || req.Cases[1].Name != beforeCase { t.Fatalf("the validator's request: %+v %v", req, err) } + // The revision names no definitions: the request holds the trunk's seat list as seat-list handed it, named by its digest (H-A-443). + if list, digest := seatListBytes(); req.SeatList != digest { + t.Errorf("the validator's request names the seat list %q, want %q", req.SeatList, digest) + } else if got, _ := json.Marshal(req.Seats); string(got) != string(list) { + t.Errorf("the validator's request holds seats %d bytes long, not the seat list's %d", len(got), len(list)) + } v := composer.Verdict{V: composer.Version} for _, c := range req.Cases { reads := map[string]composer.ManifestRead{} @@ -365,10 +391,8 @@ func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T if e := ask(8, "web", validator); e.Outcome != meshwants.StepEndedOutcomePassed { t.Errorf("the validate step of a manifest with no problem ended %+v", e) } - if e := ask(9, "a", nil); e.Outcome != meshwants.StepEndedOutcomeFailed || e.Said != reasonCycle+": a:x → b:y → a:x" { - t.Errorf("the validate step of a take-in making a cycle ended %+v", e) - } if info, err := worker.Info(b.ctx); err != nil || info.NumPending != 0 { t.Errorf("a validator is asked for the cycle: %+v %v", info, err) } + queueEmpty(b.ctx, t, b.js) // every ask acknowledged at its end (§777) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go index ee2541c18..d38980356 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go @@ -3,15 +3,115 @@ package main import ( "context" "encoding/json" + "errors" + "fmt" "slices" "strings" "sync" "testing" "time" + "github.com/nats-io/nats.go" + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + stdio "git.novox.be/novox/mesh-sdk/go" ) +// fakeMsg is one ask pulled from the seat's work queue, recording how it was settled. +type fakeMsg struct { + data []byte + h nats.Header + mu sync.Mutex + acks, naks, terms, inProgresss int +} + +func (m *fakeMsg) Data() []byte { return m.data } +func (m *fakeMsg) Headers() nats.Header { return m.h } +func (m *fakeMsg) count(n *int) error { m.mu.Lock(); *n++; m.mu.Unlock(); return nil } +func (m *fakeMsg) Ack() error { return m.count(&m.acks) } +func (m *fakeMsg) Nak() error { return m.count(&m.naks) } +func (m *fakeMsg) Term() error { return m.count(&m.terms) } +func (m *fakeMsg) InProgress() error { return m.count(&m.inProgresss) } +func (m *fakeMsg) settled() (acks, naks, terms, progress int) { + m.mu.Lock() + defer m.mu.Unlock() + return m.acks, m.naks, m.terms, m.inProgresss +} + +// fakeTaken is the state taken: its keys, each with its time to live, and a refusal of TakeOnce when refuse is set. +type fakeTaken struct { + mu sync.Mutex + values map[string][]byte + ttls map[string]time.Duration + refuse error +} + +func newFakeTaken() *fakeTaken { + return &fakeTaken{values: map[string][]byte{}, ttls: map[string]time.Duration{}} +} + +func (f *fakeTaken) TakeOnce(seat, ask string, bound time.Duration) (bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.refuse != nil { + return false, f.refuse + } + key := seat + "." + ask + if _, ok := f.values[key]; ok { + return false, nil + } + f.values[key], f.ttls[key] = []byte(`{}`), bound + return true, nil +} + +func (f *fakeTaken) Get(key string) (*stdio.StateEntry, error) { + f.mu.Lock() + defer f.mu.Unlock() + v, ok := f.values[key] + if !ok { + return nil, nil + } + return &stdio.StateEntry{Key: key, Value: v}, nil +} + +func (f *fakeTaken) RenewTTL(key string, value any, ttl time.Duration) (uint64, error) { + raw, err := json.Marshal(value) + f.mu.Lock() + defer f.mu.Unlock() + f.values[key], f.ttls[key] = raw, ttl + return 1, err +} + +func (f *fakeTaken) Delete(key string) error { + f.mu.Lock() + defer f.mu.Unlock() + delete(f.values, key) + return nil +} + +func (f *fakeTaken) Keys() ([]string, error) { + f.mu.Lock() + defer f.mu.Unlock() + var out []string + for k := range f.values { + out = append(out, k) + } + return out, nil +} + +// pullOne hands c one ask pulled from the seat's work queue under id, with a state taken of its own when c has none. +func pullOne(c *compositionSteps, id string, body []byte) *fakeMsg { + if c.taken == nil { + c.taken = newFakeTaken() + } + m := &fakeMsg{data: body, h: nats.Header{}} + if id != "" { + m.h.Set("x-event-id", id) + } + c.pulled(context.Background(), m) + return m +} + // saidEvents keeps what a publisher says. type saidEvents struct { mu sync.Mutex @@ -38,24 +138,34 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) { said := newSaidEvents() at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC) var asked meshwants.StepRun - c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf, + var lines []string + var linesMu sync.Mutex + logf := func(f string, a ...any) { + linesMu.Lock() + lines = append(lines, fmt.Sprintf(f, a...)) + linesMu.Unlock() + } + c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: logf, run: func(_ context.Context, id string, run meshwants.StepRun) stepOutcome { asked = run return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"}, Shared: []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}}} }} other, _ := json.Marshal(map[string]any{"pipeline": "main.4.m", "step": "build", "module": "m"}) - if c.take(context.Background(), "ask-0", other) { + if m := pullOne(c, "ask-0", other); m.terms != 1 { t.Fatal("a step ask of another step is taken") } body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main", Commit: "c0ffee", Revision: 4, Fingerprint: "fp1", Rerun: true, By: "lead@n1"}) - if c.take(context.Background(), "", body) { + if m := pullOne(c, "", body); m.terms != 1 { t.Fatal("a step ask with no id is taken") } - if !c.take(context.Background(), "ask-1", body) { + if m := pullOne(c, "ask-1", body); m.terms != 0 { t.Fatal("the composition step is not taken") } + if e := <-said.heard; e != meshwants.EventAskTaken+".ask-1" { // said at the pull (H2 §777) + t.Fatalf("said %s first, want ask-taken", e) + } select { case <-said.heard: case <-time.After(5 * time.Second): @@ -73,6 +183,211 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) { len(end.Shared) != 1 || end.Shared[0] != (meshwants.Shared{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}) { // §723, §729 t.Fatalf("step-ended %+v", end) } + // Design 47 H2 §792: the ask answered is one journal line, said where its end is said. + linesMu.Lock() + answered := slices.DeleteFunc(slices.Clone(lines), func(l string) bool { return !strings.HasPrefix(l, "answered ") }) + linesMu.Unlock() + if !slices.Equal(answered, []string{"answered ask-1 main.4.m m composition failed"}) { + t.Errorf("the journal says %q, want one answered line", answered) + } + // Design 47 H2 §777, §775 (H-A-381, H-A-401, H-A-404): the asks of the seat's work queue, taken once by id, said ask-taken at + // the pull, in progress while they run, acknowledged once their end is said, and ended by the seat's drop. + t.Run("work queue", queueCases) +} + +func queueCases(t *testing.T) { + at := time.Date(2026, 11, 22, 1, 0, 0, 0, time.UTC) + body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepValidate, Module: "m", Branch: "main", Revision: 4}) + type bed struct { + c *compositionSteps + said *saidEvents + taken *fakeTaken + runs chan string + release chan struct{} + tick chan time.Time + } + newBed := func() *bed { + b := &bed{said: newSaidEvents(), taken: newFakeTaken(), runs: make(chan string, 8), release: make(chan struct{}), tick: make(chan time.Time)} + b.said.heard = make(chan string, 16) + run := func(ctx context.Context, id string, _ meshwants.StepRun) stepOutcome { + b.runs <- id + select { + case <-b.release: + case <-ctx.Done(): + } + return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed} + } + b.c = &compositionSteps{node: "n1", say: b.said.say, now: func() time.Time { return at }, logf: t.Logf, taken: b.taken, takenBound: 10 * time.Minute, + validate: run, inProgress: func() time.Duration { return 20 * time.Second }, + ticks: func(d time.Duration) (<-chan time.Time, func()) { + if d == 20*time.Second { + return b.tick, func() {} + } + return make(chan time.Time), func() {} + }} + return b + } + ended := func(t *testing.T, b *bed) meshwants.StepEnded { + t.Helper() + for { + select { + case e := <-b.said.heard: + if e == meshwants.EventStepEnded+".n1" { + b.said.mu.Lock() + defer b.said.mu.Unlock() + return b.said.said[e].(meshwants.StepEnded) + } + case <-time.After(5 * time.Second): + t.Fatal("no step-ended is said") + } + } + } + settledAt := func(t *testing.T, m *fakeMsg, acks int) { + t.Helper() + deadline := time.After(5 * time.Second) + for { + if a, _, _, _ := m.settled(); a == acks { + return + } + select { + case <-deadline: + t.Fatalf("the ask is acknowledged %d times, want %d", m.acks, acks) + case <-time.After(time.Millisecond): + } + } + } + t.Run("taken once, said, in progress, acknowledged at its end", func(t *testing.T) { + b := newBed() + m := pullOne(b.c, "ask-1", body) + if id := <-b.runs; id != "ask-1" { + t.Fatalf("ran %s", id) + } + b.said.mu.Lock() + taken, ok := b.said.said[meshwants.EventAskTaken+".ask-1"].(meshwants.AskTakenEvent) + takenID := b.said.ids[meshwants.EventAskTaken+".ask-1"] + b.said.mu.Unlock() + if !ok || taken.Ask != "ask-1" || taken.Node != "n1" || !taken.At.Equal(at) || takenID != "ask-1.taken" || b.taken.ttls["mesh-wants.ask-1"] != 10*time.Minute { + t.Fatalf("ask-taken %+v under %q, taken %v", taken, takenID, b.taken.ttls) + } + b.tick <- at + b.tick <- at + // handed again while it runs: held for the run, which answers both at its one end + again := pullOne(b.c, "ask-1", body) + if a, n, _, p := m.settled(); a != 0 || n != 0 || p < 1 { + t.Fatalf("before its end the ask is acked %d, nakked %d, in progress %d times", a, n, p) + } + close(b.release) + if e := ended(t, b); e.ID != "ask-1" || e.Outcome != meshwants.StepEndedOutcomePassed { + t.Fatalf("step-ended %+v", e) + } + settledAt(t, m, 1) + settledAt(t, again, 1) + select { + case id := <-b.runs: + t.Fatalf("the ask handed again ran again (%s)", id) + default: + } + }) + t.Run("taken before is run again, said no ask-taken", func(t *testing.T) { + b := newBed() + _, _ = b.taken.TakeOnce(meshwants.Seat, "ask-2", time.Minute) + close(b.release) + m := pullOne(b.c, "ask-2", body) + <-b.runs + ended(t, b) + settledAt(t, m, 1) + b.said.mu.Lock() + _, said := b.said.said[meshwants.EventAskTaken+".ask-2"] + b.said.mu.Unlock() + if said { + t.Fatal("ask-taken is said again for an ask taken before") + } + }) + t.Run("a refused take is handed again", func(t *testing.T) { + b := newBed() + b.taken.refuse = errors.New("no bus") + m := pullOne(b.c, "ask-3", body) + if _, n, _, _ := m.settled(); n != 1 || len(b.runs) != 0 { + t.Fatalf("nakked %d, ran %d", n, len(b.runs)) + } + }) + t.Run("dropped while it runs", func(t *testing.T) { + b := newBed() + m := pullOne(b.c, "ask-4", body) + <-b.runs + if _, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-4"}); err == nil { + t.Fatal("a drop with no why is taken") + } + a, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-4", Why: "its bound passed"}) + if err != nil || a["held"] != true { + t.Fatalf("drop answered %v, %v", a, err) + } + e := ended(t, b) + if e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || e.Said != "the ask was dropped by n1.mesh-pipeline-runner: its bound passed" { + t.Fatalf("step-ended %+v", e) + } + settledAt(t, m, 1) + }) + t.Run("dropped before it was pulled", func(t *testing.T) { + b := newBed() + a, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-5", By: "the runner", Why: "its bound passed"}) + if err != nil || a["held"] != false || b.taken.ttls["dropped.ask-5"] != 10*time.Minute { + t.Fatalf("drop answered %v, %v; taken %v", a, err, b.taken.ttls) + } + m := pullOne(b.c, "ask-5", body) + e := ended(t, b) + if e.ID != "ask-5" || e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || e.Said != "the ask was dropped by the runner: its bound passed" { + t.Fatalf("step-ended %+v", e) + } + settledAt(t, m, 1) + if len(b.runs) != 0 { + t.Fatal("a dropped ask ran") + } + if v, _ := b.taken.Get("dropped.ask-5"); v != nil { + t.Fatal("the drop's key stays once its ask ended") + } + }) + t.Run("the asks tools", func(t *testing.T) { + taken := newFakeTaken() + _, _ = taken.TakeOnce(meshwants.Seat, "ask-6", time.Minute) + _, _ = taken.RenewTTL("dropped.ask-7", askDrop{By: "the runner", Why: "its bound passed", At: at}, time.Minute) + waiting := func(seat string) (stdio.Waiting, error) { + if seat != meshwants.Seat { + t.Errorf("the waiting asks of %s are read", seat) + } + return stdio.Waiting{Asks: []stdio.WaitingAsk{{ID: "ask-8", Module: "mesh-delivery-pipeline", AgeSeconds: 3}, + {ID: "ask-6", Module: "mesh-delivery-pipeline", Again: true}, {ID: "ask-9", Module: "mesh-delivery-pipeline", Again: true}}}, nil + } + var asked []string + ask := func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + asked = append(asked, key+" "+string(raw)) + if strings.HasSuffix(key, "."+meshwants.VerbDrop) { + return json.RawMessage(`{"held":false}`), nil + } + return json.RawMessage(`{}`), nil + } + tools := map[string]stdio.Tool{} + for _, tool := range asksTools(waiting, taken, ask) { + tools[tool.Name] = tool + } + out, err := tools[asksTool].Run(map[string]any{}) + a, _ := out.(map[string]any) + w, _ := a["waiting"].([]map[string]any) + d, _ := a["dropped"].([]map[string]any) + if err != nil || len(w) != 3 || w[0]["id"] != "ask-8" || w[0]["state"] != "waiting" || w[1]["state"] != "held" || w[2]["state"] != "waiting again" || !slices.Equal(a["taken"].([]string), []string{"ask-6"}) || len(d) != 1 || + d[0]["id"] != "ask-7" || d[0]["by"] != "the runner" || a["total"] != 5 || a["capped"] != false { + t.Fatalf("assigner_asks answered %+v, %v", out, err) + } + if _, err := tools[askDropTool].Run(map[string]any{"ask": "ask-8"}); err == nil || len(asked) != 0 { + t.Fatalf("a drop with no why: %v, asked %v", err, asked) + } + out, err = tools[askDropTool].Run(map[string]any{"ask": "ask-8", "why": "stuck"}) + if err != nil || len(asked) != 2 || !strings.HasPrefix(asked[0], "seat:mesh-hand-acts.hand-act ") || !strings.Contains(asked[0], askDropTool+" ask-8") || + !strings.HasPrefix(asked[1], "seat:mesh-wants.drop ") || !strings.Contains(asked[1], `"id":"ask-8"`) || !strings.Contains(asked[1], `"why":"stuck"`) { + t.Fatalf("assigner_ask_drop answered %v, %v; asked %v", out, err, asked) + } + }) } // Design 47 G §496, A-262: a running composition step says step-progress. on every tick of its ticker, with the ask, its @@ -98,9 +413,10 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) { }} // A11/AC1 (design 47 H §752, H-A-88): with no callers in mesh-wants, the progress names the asker the step run carries. body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main", Asker: "mesh-pipeline-runner"}) - if !c.take(context.Background(), "ask-1", body) { + if m := pullOne(c, "ask-1", body); m.terms != 0 { t.Fatal("the composition step is not taken") } + <-said.heard // ask-taken, at the pull (H2 §777) progress := meshwants.EventStepProgress + ".n1" for range 2 { tick <- at @@ -149,7 +465,7 @@ func TestASupersededStepLeavesItsRuns(t *testing.T) { }} body := func(s meshwants.StepRun) []byte { raw, _ := json.Marshal(s); return raw } for _, p := range []string{"main.3.a", "main.3.b"} { - if !c.take(context.Background(), "ask-"+p, body(step(p))) { + if m := pullOne(c, "ask-"+p, body(step(p))); m.terms != 0 { t.Fatal("not taken") } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go index b60771eb2..1eb4cc98e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go @@ -5,18 +5,25 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "slices" "strings" "sync" "time" - mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + mcatc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-assigner/internal/assign" "mesh-delivery-assigner/internal/assigngraph" + "mesh-delivery-assigner/internal/assignment" ) // graphKeeper is the pass's assignment graph (design 47 G §292, A-67): per catalog revision taken in it computes the revision @@ -28,7 +35,7 @@ type graphKeeper struct { // reads is the same store through a handle of the pass's own, for the pass's read of its fill (graphfill.go); nil reads objects. reads assigngraph.Objects // halves is every followed branch's catalog graph revision held now. - halves func() map[string]mcc.Half + halves func() map[string]mcatc.Half // mu guards what the keeping writes and the pass reads: the revision kept, full, and the need handed over (design 47 H-A-259). mu sync.Mutex // handedRev is the revision last handed to the keeping, so a pass whose inputs did not change hands nothing; handed is false @@ -46,14 +53,40 @@ type graphKeeper struct { full, started bool // revisions removes a revision that ceased to be current (revisions.go, H-A-86); nil keeps every revision. revisions *revisionRemover - // computed is the revision computed by the last pass, with its entries: its edges. - computed assigngraph.Computed + // computed is the revision computed by the last pass, with its entries: its edges; hasComputed says a pass computed one. + computed assigngraph.Computed + hasComputed bool + // keepMu runs one keeping at a time, the pass's (keepOne) or one by hand (keepNow). + keepMu sync.Mutex // The keeping off the pass loop (H-A-259): need is what waits to be kept, keeping says one runs; background runs it (nil: at // once, in the caller), and wake makes a pass once a keeping changed what the pass reads. need *keepNeed keeping bool background func(func()) wake func() + // A keeping that fails runs again after retry, keeping-retry, by after (time.AfterFunc; a test hands its own); failures + // counts the failures in a row, and keeping-failed is raised at the third (design 47 H2 §813 (2)). + retry time.Duration + after func(d time.Duration, f func()) + failures int +} + +// The keeping's setting and finding (design 47 H2 §813 (2); ADR 0682 §11: the holder's own deadline). +const ( + keepingRetrySetting = "keeping-retry" + defaultKeepingRetry = 30 * time.Second + kindKeepingFailed = "keeping-failed" + // keepingFailedAt is the failures in a row that raise keeping-failed. + keepingFailedAt = 3 +) + +// keepingRetry reads the setting keeping-retry (§781). +func keepingRetry() (time.Duration, error) { + d, err := stdio.SettingDuration(keepingRetrySetting, defaultKeepingRetry) + if err != nil { + return 0, errors.New("the assigner's " + err.Error()) + } + return d, nil } // keepNeed is one need of the keeping: a computed revision to keep (nil: none), and whether the revisions not current are judged. @@ -99,7 +132,9 @@ func (p *passer) keepGraph(ctx context.Context) string { return "" } c := assigngraph.Compute(p.graphInputs()) - g.computed = c + g.mu.Lock() + g.computed, g.hasComputed = c, true + g.mu.Unlock() for _, u := range c.Uncomputed { if g.said != u { g.said = u @@ -177,6 +212,8 @@ func (p *passer) needKeeping(ctx context.Context, n keepNeed) { // revisions that ceased to be current; it says its duration in the journal, and wakes a pass when it changed what the pass reads. func (p *passer) keepOne(ctx context.Context, n keepNeed) { g := p.graph + g.keepMu.Lock() + defer g.keepMu.Unlock() start := time.Now() changed := false defer func() { @@ -199,6 +236,12 @@ func (p *passer) keepOne(ctx context.Context, n keepNeed) { // keepRevision keeps c when the store holds it, and says whether a revision was kept now. func (p *passer) keepRevision(ctx context.Context, c assigngraph.Computed) bool { + fresh, _ := p.keepRevisionOf(ctx, c) + return fresh +} + +// keepRevisionOf is keepRevision answering the fault the keeping met too. +func (p *passer) keepRevisionOf(ctx context.Context, c assigngraph.Computed) (bool, error) { g := p.graph again := func() { // the next pass hands this revision again g.mu.Lock() @@ -208,10 +251,11 @@ func (p *passer) keepRevision(ctx context.Context, c assigngraph.Computed) bool used, _, _ := g.fillOf(ctx) rev, fresh, err := assigngraph.Keep(ctx, g.objects, c, meshwants.EventAssignmentRevision, p.clock()) if err != nil { - p.logf("the assignment graph revision is not kept, so no want is said as wanted until it is: %v", err) again() - return false + p.keepingFailed(ctx, c, err) + return false, err } + p.keepingSucceeded() var source int64 for _, cat := range rev.Catalogs { if cat.Branch == p.w.trunkName() { @@ -222,14 +266,54 @@ func (p *passer) keepRevision(ctx context.Context, c assigngraph.Computed) bool g.kept, g.ok, g.source = rev, true, source g.mu.Unlock() if !fresh { - return false + return false, nil } p.fillingAfter(ctx, used) id := fmt.Sprintf("%s-%s-%d", meshwants.EventAssignmentRevision, rev.Kind, rev.Revision) if err := p.wantsSay(meshwants.EventAssignmentRevision, assigngraph.Event(rev), id); err != nil { p.logf("the assignment revision %d is kept but not said: %v", rev.Revision, err) } - return true + return true, nil +} + +// keepNow is the seat mesh-wants' act keep-now (design 47 H2 §813 (2); H-A-417): the keeping of the pass's newest computed +// revision, run at once, one at a time with the pass's; it answers the revision kept, or the fault the keeping met (a failed +// one is run again after keeping-retry, as every keeping is). The hand act is the tool's to write, before it calls. +func (p *passer) keepNow(ctx context.Context, in meshwants.KeepNowInput) (meshwants.KeepNowAnswer, error) { + g := p.graph + if strings.TrimSpace(in.Why) == "" { + return meshwants.KeepNowAnswer{}, errors.New("say why the keeping runs now") + } + if g == nil || g.objects == nil { + return meshwants.KeepNowAnswer{}, errors.New("the assigner keeps no assignment graph here") + } + g.mu.Lock() + c, ok := g.computed, g.hasComputed + g.mu.Unlock() + if !ok { + return meshwants.KeepNowAnswer{}, errors.New("no pass has computed an assignment graph revision yet") + } + g.keepMu.Lock() + defer g.keepMu.Unlock() + p.logf("the keeping of the assignment graph runs now by hand: %s", in.Why) + if _, err := p.keepRevisionOf(ctx, c); err != nil { + return meshwants.KeepNowAnswer{Fault: err.Error()}, nil + } + rev, _, _ := g.keptNow() + return meshwants.KeepNowAnswer{Revision: &rev}, nil +} + +// keepNowVerb is the process's handler of the seat mesh-wants' act keep-now. +func keepNowVerb(ctx context.Context, p *passer) map[string]process.SeatHandler { + return map[string]process.SeatHandler{ + meshwants.VerbKeepNow: func(_ string, raw json.RawMessage) (any, error) { + var in meshwants.KeepNowInput + if err := json.Unmarshal(raw, &in); err != nil { + return nil, fmt.Errorf("the arguments are not JSON: %w", err) + } + return p.keepNow(ctx, in) + }, + } } // graphInputs are what the pass's assignment graph is computed from: every followed branch's catalog half, the placements, each @@ -366,3 +450,63 @@ func (p *passer) graphKept() (meshwants.AssignmentRevision, int64, bool) { } return p.graph.keptNow() } + +// keepingFailed marks the keeping of c wanted again after keeping-retry (design 47 H2 §813 (2)): run then unless a newer +// revision was handed meanwhile, whose own keeping runs. The third failure in a row raises keeping-failed (scope module). +func (p *passer) keepingFailed(ctx context.Context, c assigngraph.Computed, err error) { + g := p.graph + g.mu.Lock() + g.failures++ + n, retry, after := g.failures, g.retry, g.after + g.mu.Unlock() + if retry <= 0 { + retry = defaultKeepingRetry + } + if after == nil { + after = func(d time.Duration, f func()) { time.AfterFunc(d, f) } + } + p.logf("the assignment graph revision is not kept (%d in a row), so no want is said as wanted until it is; kept again in %s: %v", n, retry, err) + if n == keepingFailedAt && p.findings.Put != nil { + if err := p.findings.Put(keepingFailedFinding(n, err).Finding(p.clock())); err != nil { + p.logf("the finding %s is not written: %v", kindKeepingFailed, err) + } + } + after(retry, func() { + g.mu.Lock() + newer := g.handed && !assigngraph.Same(g.handedRev, c.Revision) + if !newer { + g.handedRev, g.handed = c.Revision, true + } + g.mu.Unlock() + if !newer { + p.needKeeping(ctx, keepNeed{c: &c}) + } + }) +} + +// keepingSucceeded ends a run of failures: keeping-failed, raised at the third, is cleared. +func (p *passer) keepingSucceeded() { + g := p.graph + g.mu.Lock() + n := g.failures + g.failures = 0 + g.mu.Unlock() + if n >= keepingFailedAt && p.findings.Delete != nil { + o := keepingFailedFinding(n, nil) + if err := p.findings.Delete(views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)); err != nil { + p.logf("the finding %s is not deleted: %v", kindKeepingFailed, err) + } + } +} + +// keepingFailedFinding is keeping-failed of the assigner on this node, after n failures in a row (design 47 H2 §813 (2)). +func keepingFailedFinding(n int, err error) assign.Observed { + summary := fmt.Sprintf("the keeping of the assignment graph revisions failed %d times in a row", n) + if err != nil { + summary += ": " + err.Error() + } + return assign.Observed{Scope: mcc.ScopeModule, ID: assignment.Module() + "." + thisNode, Kind: kindKeepingFailed, + Severity: mcseat.FindingSeverityWarning, Summary: summary, Headline: "The assignment graph is not kept", + Explanation: summary + ". No want is said as wanted until a keeping succeeds; it is run again after keeping-retry.", + Source: "assigner." + kindKeepingFailed} +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go index 5acdb4241..fdf8d4fd0 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "slices" "strings" @@ -11,6 +12,7 @@ import ( "time" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -18,6 +20,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/catalogue" + "mesh-delivery-assigner/internal/assign" "mesh-delivery-assigner/internal/assigngraph" "mesh-delivery-assigner/internal/buckets" ) @@ -92,6 +95,35 @@ func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) { if err := json.Unmarshal(events[len(events)-1].body, &w); err != nil || len(events) != 3 || !w.Removed || w.Fp != fpTest { t.Errorf("a removing want said %+v (%v), %d events", w, err, len(events)) } + // Design 47 H2 §813 (1), H-A-416: a want carries its run, its pipeline's source in decimal, or the newest source held for a + // want of no pipeline; it is written by compare-and-set refusing a lower run, and one that differs only by its run stands. + t.Run("a want's run", func(t *testing.T) { + wkey, _ := mwc.WantKey("g14", "db") + stored := func() mwc.Want { w, _ := mwc.DecodeWant(kv.values[wkey]); return w } + p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1}) + if w := stored(); w.Run != "12" || w.Pipeline != "main.12" { + t.Fatalf("the want of pipeline main.12 is %+v, want run 12", w) + } + puts := kv.puts + p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.1", Pipeline: "main.11", Rank: 1}) + if w := stored(); w.Run != "12" || w.Version != "0.1.2" || kv.puts != puts { + t.Errorf("a want of a lower run replaced %+v (%d writes)", w, kv.puts-puts) + } + p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1}) + if kv.puts != puts { + t.Error("a want that stands was written again") + } + skey, _ := mwc.WantKey("shanks", "db") + p.putWant(ctx, "shanks", "db", mwc.Want{Fp: fpTest, Put: true, Words: "put: by hand"}) + if w, _ := mwc.DecodeWant(kv.values[skey]); w.Run != "9" { + t.Errorf("a put's want is %+v, want the newest source held, 9", w) + } + // a placement's build after the delivery of run 12: the newest source held for it is the want's own, never refused + p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.0"}) + if w := stored(); w.Run != "12" || w.Version != "0.1.0" { + t.Errorf("a placement's want after run 12 is %+v, want it written at run 12", w) + } + }) // A-201, A-235: a run's candidate catalog revision, read whole through the seat's verb revision, keeps the run's assignment // graph under candidate.. in place of its branch's half; it says nothing and writes no want, a try's candidate @@ -273,4 +305,65 @@ func TestAPassWritesWantsWhileAKeepingIsBlocked(t *testing.T) { if !said("the pass took ") { t.Error("the pass does not say its duration") } + open() + kept.Wait() + // Design 47 H2 §813 (2): a keeping that fails runs again after keeping-retry with no pass, raising keeping-failed at the third + // failure in a row, cleared at the next that succeeds. + t.Run("a failed keeping runs again", func(t *testing.T) { + failing := &failingObjects{blockedObjects: objects, fail: 3} + var retries []time.Duration + var put []mcseat.Finding + var deleted []string + p.findings = assign.Findings{Put: func(f mcseat.Finding) error { put = append(put, f); return nil }, + Delete: func(key string) error { deleted = append(deleted, key); return nil }} + p.graph.objects, p.graph.background, p.graph.retry = failing, nil, 30*time.Second + p.graph.after = func(d time.Duration, f func()) { retries = append(retries, d); f() } + placeOn(t, p.w, "shanks", "web") // new inputs: a keeping is handed + pass() + if len(retries) != 3 || retries[0] != 30*time.Second || len(put) != 1 || put[0].Kind != kindKeepingFailed || put[0].Scope != "module" || + len(deleted) != 1 || !strings.Contains(deleted[0], kindKeepingFailed) || p.graph.failures != 0 { + t.Fatalf("retries %v, raised %+v, cleared %v, failures %d", retries, put, deleted, p.graph.failures) + } + if _, _, ok := p.graph.keptNow(); !ok || failing.fail != 0 { + t.Fatal("the revision is not kept once the store takes it") + } + // keep-now (H-A-417): the keeping by hand answers the revision kept, or the fault met; it needs a why. + if _, err := p.keepNow(context.Background(), meshwants.KeepNowInput{}); err == nil { + t.Error("a keeping by hand with no why runs") + } + if a, err := p.keepNow(context.Background(), meshwants.KeepNowInput{Why: "the store was full"}); err != nil || a.Revision == nil || a.Fault != "" { + t.Errorf("keep-now answered %+v, %v; want the revision kept", a, err) + } + failing.fail, retries = 2, nil + p.graph.after = func(d time.Duration, _ func()) { retries = append(retries, d) } + placeOn(t, p.w, "novox", "web") // a new revision, whose keeping the store refuses at the pass and again by hand + pass() + if a, err := p.keepNow(context.Background(), meshwants.KeepNowInput{Why: "again"}); err != nil || a.Revision != nil || !strings.Contains(a.Fault, "the store answers nothing") || len(retries) != 2 { + t.Errorf("keep-now on a refusing store answered %+v, %v, retries %v; want the fault and a retry", a, err, retries) + } + var asked []string + tool := keepNowTools(func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + asked = append(asked, key+" "+string(raw)) + return json.RawMessage(`{}`), nil + })[0] + if _, err := tool.Run(map[string]any{"why": "by hand"}); err != nil || len(asked) != 2 || !strings.HasPrefix(asked[0], "seat:mesh-hand-acts.hand-act ") || + asked[1] != `seat:mesh-wants.keep-now {"why":"by hand"}` { + t.Errorf("assigner_keep_now: %v, asked %v", err, asked) + } + }) +} + +// failingObjects refuses the next fail puts, then is objects. +type failingObjects struct { + *blockedObjects + fail int +} + +func (f *failingObjects) Put(ctx context.Context, name string, data []byte) error { + if f.fail > 0 { + f.fail-- + return errors.New("the store answers nothing") + } + return f.blockedObjects.Put(ctx, name, data) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 1f67ede76..8358c76d8 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -50,6 +50,7 @@ import ( mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/announce" + "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/process" @@ -177,7 +178,7 @@ func startBed(t *testing.T, leaveOut string) *bed { continue } cfg := jetstream.KeyValueConfig{Bucket: name, History: 5} - if name == plc.PulsesBucket() || name == buckets.CompositionRuns() { + if name == plc.PulsesBucket() || name == buckets.CompositionRuns() || name == buckets.Taken() { cfg.History, cfg.LimitMarkerTTL = 1, time.Minute // a time to live per key, as the controller makes it (key-ttl) } kv, err := js.CreateKeyValue(ctx, cfg) @@ -206,6 +207,9 @@ func startBed(t *testing.T, leaveOut string) *bed { if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil { t.Fatal(err) } + // The seat mesh-wants' work queue and its one worker, as the broker derives them from the seat's definition (design 47 H2 + // §777); the assigner pulls its step asks from it. + seatQueue(ctx, t, js) // The node-engine seat's stream of `running`, kept last per subject, which the assigner hears the health of what runs from (§520). if _, err := js.CreateStream(ctx, keptStream(nodeengine.StreamRunning, nodeengine.SubjectRunningAll, jetstream.StreamConfig{MaxMsgsPerSubject: 1})); err != nil { t.Fatal(err) @@ -1938,7 +1942,9 @@ func awaitKept(t *testing.T, b *bed, stream string, n uint64) { // serveSeatList answers the seat mesh-catalog's verb seat-definitions as the indexer does (ADR 0636 §4): with no seat, every // trunk seat with its digest; with one, its definition resolved, from the mesh-sdk test copy of every definition. The channel -// it answers is closed once every listed seat's definition was asked, the end of a reader's first read. +// it answers is closed once every listed seat's definition was asked, the end of a reader's first read. It also answers the +// chunked read verb seat-list as the indexer does (design 47 H2 H-A-443): the definitions as one JSON array sorted by seat, +// its sha256 and size, then the bytes on the call's chunk subjects (seatListBytes). func serveSeatList(t *testing.T, nc *nats.Conn) <-chan struct{} { t.Helper() var mu sync.Mutex @@ -1981,5 +1987,94 @@ func serveSeatList(t *testing.T, nc *nats.Conn) <-chan struct{} { } t.Cleanup(func() { _ = sub.Unsubscribe() }) } + list, digest := seatListBytes() + hand := func(m *nats.Msg) { + var in meshcatalog.SeatListInput + _ = json.Unmarshal(m.Data, &in) + send := buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbSeatList, Call: in.Call} + chunks := int64((len(list) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes) + raw, _ := json.Marshal(map[string]any{"result": meshcatalog.SeatListAnswer{Digest: digest, Size: int64(len(list)), Chunks: chunks}, "node": "n0"}) + _ = m.Respond(raw) + go func() { _ = send.Send(nc, list, buscalls.SeatAckWithin, t.Logf) }() + } + for _, subject := range []string{"mesh.seat." + meshcatalog.Seat + ".tool." + meshcatalog.VerbSeatList, "mesh.seat." + meshcatalog.Seat + ".call." + meshcatalog.VerbSeatList + ".*"} { + sub, err := nc.Subscribe(subject, hand) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + } return read } + +// seatListBytes are the bed's seat list as the verb seat-list hands it: every definition of the mesh-sdk test copy as one JSON +// array sorted by seat, and its sha256 (H-A-441 (a)). +func seatListBytes() ([]byte, string) { + defs := slices.Clone(seatlisttest.Definitions()) + slices.SortFunc(defs, func(a, b seatdef.Definition) int { return strings.Compare(a.Seat, b.Seat) }) + list := make([]json.RawMessage, 0, len(defs)) + for _, d := range defs { + raw, _ := json.Marshal(d) + list = append(list, raw) + } + raw, _ := json.Marshal(list) + return raw, seatListDigest(raw) +} + +// seatQueue makes the seat mesh-wants' work queue and its one shared worker from the seat's definition, as the broker derives +// them (design 47 H2 §777). +func seatQueue(ctx context.Context, t *testing.T, js jetstream.JetStream) { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "..", "mesh-delivery-contracts", "seats", meshwants.Seat, "seat.json")) + if err != nil { + t.Fatal(err) + } + def, err := seatdef.Parse(raw) + if err != nil { + t.Fatal(err) + } + stream, worker, ok, err := buscompose.SeatQueue(meshwants.Seat, def.Accepts) + if err != nil || !ok { + t.Fatalf("the seat's work queue: %v %v", ok, err) + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream.Name, Subjects: stream.Subjects, Retention: jetstream.WorkQueuePolicy, + MaxAge: time.Duration(stream.MaxAge) * time.Second, MaxMsgs: int64(stream.MaxMsgs), Duplicates: time.Duration(stream.DuplicatesSeconds) * time.Second}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateOrUpdateConsumer(ctx, stream.Name, jetstream.ConsumerConfig{Durable: worker.Name, FilterSubjects: worker.Filters, AckPolicy: jetstream.AckExplicitPolicy, + AckWait: time.Duration(worker.AckWaitSeconds) * time.Second, MaxDeliver: worker.MaxDeliver, MaxAckPending: worker.MaxAckPending}); err != nil { + t.Fatal(err) + } +} + +// putStep puts one step ask on the seat mesh-wants' work queue under id, as the runner puts it (design 47 H2 §777). +func putStep(ctx context.Context, t *testing.T, js jetstream.JetStream, id string, step meshwants.StepRun) { + t.Helper() + msg := nats.NewMsg(meshwants.AcceptSubjectStep) + msg.Header.Set("x-event-id", id) + msg.Header.Set(nats.MsgIdHdr, id) + msg.Data, _ = json.Marshal(step) + if _, err := js.PublishMsg(ctx, msg); err != nil { + t.Fatalf("the ask %s is not put on the seat's work queue: %v", id, err) + } +} + +// queueEmpty fails t unless the seat mesh-wants' worker holds no ask, waiting or unacknowledged (design 47 H2 §777). +func queueEmpty(ctx context.Context, t *testing.T, js jetstream.JetStream) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + c, err := js.Consumer(ctx, buscompose.SeatQueueStreamName(meshwants.Seat), buscompose.SeatQueueWorkerName(meshwants.Seat)) + if err != nil { + t.Fatal(err) + } + info, err := c.Info(ctx) + if err == nil && info.NumPending == 0 && info.NumAckPending == 0 { + return + } + if time.Now().After(deadline) { + t.Fatalf("the seat's worker holds asks after their ends: %+v %v", info, err) + } + time.Sleep(20 * time.Millisecond) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 7c6230ac3..d0fff5108 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -14,8 +14,10 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/buscompose" "io" "log" + "maps" "mesh-delivery-assigner/internal/assigngraph" "mesh-delivery-assigner/internal/assignment" "mesh-delivery-assigner/internal/buckets" @@ -23,6 +25,7 @@ import ( "os/signal" "slices" "strings" + "sync/atomic" "syscall" "time" @@ -89,7 +92,7 @@ func ownBuckets() []string { return []string{buckets.Asks(), buckets.Differences(), buckets.Puts(), buckets.Wants(), plc.AssignmentsBucket(), plc.ProposalsBucket(), plc.CandidatesBucket(), plc.PulsesBucket(), assign.FindingsBucket(), assign.PoliciesBucket(), plc.SeatHoldersBucket(), - engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(), buckets.CompositionRuns()} + engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(), buckets.CompositionRuns(), buckets.Taken()} } // engineReadsState is the assigner's state of what each node's engine says it reads (design 47 H-A-104), one key `` @@ -136,6 +139,7 @@ var keptStates = []announce.State{ {Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211) {Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354) {Name: buckets.CompositionRunsState, Readable: true}, // the composition runs, read by the process and its tools (design 47 G §719, A-468c) + {Name: process.TakenState, Readable: true}, // the asks of the seat mesh-wants taken once, and their drops (design 47 H2 §775, §777) } // kvSet is every bucket the assigner opened, by name. @@ -615,7 +619,11 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, objects := newBusObjects(js, process.Account{Node: thisNode, Module: assignment.Module()}, assigngraph.RevisionsObjects) // The keeping and removing of revisions runs off the pass loop, and a keeping that changed what the pass reads wakes // one (design 47 H-A-259). - p.graph = &graphKeeper{objects: objects, halves: iv.h.views, background: func(f func()) { go f() }, wake: wake, + retry, err := keepingRetry() + if err != nil { + return err + } + p.graph = &graphKeeper{objects: objects, halves: iv.h.views, background: func(f func()) { go f() }, wake: wake, retry: retry, reads: newBusObjects(js, process.Account{Node: thisNode, Module: assignment.Module()}, assigngraph.RevisionsObjects)} keep := func(ctx context.Context, ev meshcatalog.CatalogRevisionEvent) error { return p.keepCandidate(ctx, iv.call, ev) @@ -667,10 +675,25 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } // The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine. if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil { - run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, + askBound, err := composeAskBound() + if err != nil { + return err + } + run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFileKept, bound: askBound}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, callFile: bus.CallFileKept, trunk: trunkOf(iv, w.trunkName), newest: iv.newestOf(), watched: iv.watchedOf, assigned: w.assignedOn, assignment: w.assignmentOf, findings: findings, running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf} + // The seat's work queue (design 47 H2 §777): pulled through its one worker, each ask taken once in the state taken. + takenBound, err := askTakenBound() + if err != nil { + return err + } + var ackWait atomic.Int64 // the worker's ack wait, read when it is bound + ackWait.Store(int64(buscompose.SeatQueueDefaultAckWait)) steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume, - validate: run.validate, resumeValidate: run.resumeValidate, ended: runs.stepEnded, now: time.Now, logf: logger.Printf} + validate: run.validate, resumeValidate: run.resumeValidate, ended: runs.stepEnded, now: time.Now, logf: logger.Printf, + takenBound: takenBound, inProgress: inProgressOf(seatlist.Definition(gate.Reader, meshwants.Seat), func() time.Duration { return time.Duration(ackWait.Load()) })} + if e, ok := bus.(emitting); ok { + steps.taken = e.State(process.TakenState) + } move := steps.move(ctx) // the readers a rerun carried (§727, A-470d) runs.moved.Store(&move) // The state's watch wakes the steps waiting on a run and hears a bound's expiry; the runs standing at the start are taken up @@ -685,9 +708,14 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, _, err := meshhandactsclient.Recorder{Ask: bus.Call}.Record(q) return err } - stopVerbs, err := e.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), compositionVerbs(runs, steps, record)) + verbs := compositionVerbs(runs, steps, record) + maps.Copy(verbs, dropVerb(steps)) // the seat's act drop (§778, H-A-404) + if p.graph != nil { + maps.Copy(verbs, keepNowVerb(ctx, p)) // the keeping by hand (§813 (2), H-A-417) + } + stopVerbs, err := e.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), verbs) if err != nil { - logger.Printf("the verbs %s and %s are not answered until the assigner starts again: %v", meshwants.VerbCompositions, meshwants.VerbCompositionsDrop, err) + logger.Printf("the verbs %s, %s and %s are not answered until the assigner starts again: %v", meshwants.VerbCompositions, meshwants.VerbCompositionsDrop, meshwants.VerbDrop, err) } else { go func() { <-ctx.Done(); stopVerbs() }() } @@ -700,10 +728,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } p.ends.left.Store(&left) } - if err := serveCompositionSteps(ctx, nc, steps); err != nil { - logger.Printf("the composition step is not heard, so no composition step runs until the assigner starts again: %v", err) - } else { - logger.Printf("the composition step is heard on %s", meshwants.AcceptSubjectStep) + if steps.taken == nil { + logger.Printf("the state %s is not bound, so no composition step is pulled until the assigner starts again", process.TakenState) + } else if err := serveStepQueue(ctx, nc, js, steps, func(d time.Duration) { ackWait.Store(int64(d)) }); err != nil { + logger.Printf("the composition step is not pulled, so no composition step runs until the assigner starts again: %v", err) } } if bus != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 56aec2127..fcd40fdf1 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -7,8 +7,10 @@ import ( "errors" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "mesh-delivery-assigner/internal/buckets" + "reflect" "slices" "sort" + "strconv" "strings" "time" @@ -21,6 +23,8 @@ import ( "git.novox.be/novox/mesh-sdk/go/seatlist" "sync" + "github.com/nats-io/nats.go/jetstream" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -303,11 +307,93 @@ func (p *passer) putWant(ctx context.Context, node, module string, want mwc.Want fp = before.Fp } } - if p.written(ctx, buckets.Wants(), key, func() ([]byte, error) { return mwc.EncodeWant(want) }) { + if p.wantWritten(ctx, node, module, key, want) { p.sayWanted(node, module, fp, want.Pipeline, want.Removed) } } +// runOf is the run a want carries (design 47 H2 §813 (1); H-A-355 (1), H-A-416): the source revision of the pipeline that +// released it, `.`, in decimal; for a want of no pipeline (a put, a placement's build, a removal) the newest +// source the assigner holds for it: the trunk's source of its kept assignment graph revision, or the run of the want written +// (held) when that is newer, so a person's put or placement after a delivery is never refused as older; "" when it holds none. +func (p *passer) runOf(want mwc.Want, held *mwc.Want) string { + if i := strings.LastIndexByte(want.Pipeline, '.'); i >= 0 { + if n, err := strconv.ParseInt(want.Pipeline[i+1:], 10, 64); err == nil && n > 0 { + return strconv.FormatInt(n, 10) + } + } + var newest int64 + if p.graph != nil { + if _, source, ok := p.graph.keptNow(); ok { + newest = source + } + } + if held != nil { + newest = max(newest, runNumber(held.Run)) + } + if newest <= 0 { + return "" + } + return strconv.FormatInt(newest, 10) +} + +// runNumber is a want's run as a number, 0 for none. +func runNumber(run string) int64 { + n, _ := strconv.ParseInt(run, 10, 64) + return n +} + +// wantWritten writes the want of module on node under key by compare-and-set, carrying its run (design 47 H2 §813 (1)): a want +// that differs from the one written only by its run is not written again (its run stays, so a node never composes again for +// it), and one whose run is lower than the run written is refused, said in the journal. It answers whether it wrote. +func (p *passer) wantWritten(ctx context.Context, node, module, key string, want mwc.Want) bool { + var held *mwc.Want + if raw := p.w.value(buckets.Wants(), key); raw != nil { + if h, err := mwc.DecodeWant(raw); err == nil { + held = &h + } + } + want.Run = p.runOf(want, held) + if held != nil { + same := *held + same.Run = want.Run + if reflect.DeepEqual(same, want) { + want.Run = held.Run // it stands: nothing is written + } else if runNumber(held.Run) > runNumber(want.Run) { + p.logf("the want of %s on %s is not written: its run %q is lower than the run %q written (§813 (1))", module, node, want.Run, held.Run) + return false + } + } + value, err := mwc.EncodeWant(want) + if err != nil { + p.logf("%s %s is not written: %v", buckets.Wants(), key, err) + return false + } + if held := p.w.value(buckets.Wants(), key); held != nil && bytes.Equal(held, value) { + return false + } + kv := p.kvs[buckets.Wants()] + entry, err := kv.Get(ctx, key) + switch { + case errors.Is(err, jetstream.ErrKeyNotFound): + _, err = kv.Create(ctx, key, value) + case err == nil: + if h, derr := mwc.DecodeWant(entry.Value()); derr == nil && runNumber(h.Run) > runNumber(want.Run) { + p.logf("the want of %s on %s is not written: its run %q is lower than the run %q written (§813 (1))", module, node, want.Run, h.Run) + return false + } + _, err = kv.Update(ctx, key, value, entry.Revision()) + } + if err != nil { + if !errors.Is(err, context.Canceled) { + p.logf("%s %s could not be written: %v; the next pass writes it again", buckets.Wants(), key, err) + } + return false + } + p.w.put(buckets.Wants(), key, value) + return true +} + // staleAsk says why the hold, restore or record ask at key (read as the SDK reads it) is about something the trunk has left // behind, or "" when it is not: a module no longer on the trunk, a fingerprint that is no longer the module's trunk fingerprint, // or, for a record ask, a composition the module's node no longer waits on. A backup ask, and a revert ask (design 47 G §710), is stale when the module's want on its diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index cfed17a95..16c9afce9 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-assigner/internal/buckets" "mesh-delivery-assigner/internal/puts" "os" @@ -56,6 +57,7 @@ func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error) func (m *memKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) { m.values[key] = value + m.puts++ // a write that reaches the bucket (the wants' compare-and-set, design 47 H2 §813 (1)) return uint64(len(m.values)), nil } @@ -252,6 +254,7 @@ func (m *memKV) Update(_ context.Context, key string, value []byte, revision uin return 0, errors.New("wrong last sequence") } m.values[key] = value + m.puts++ return uint64(len(value)), nil } @@ -295,6 +298,9 @@ func (b *fakeBus) CallOn(node, key string, body any) (json.RawMessage, error) { } func (b *fakeBus) CallKept(key string, body any) (json.RawMessage, error) { return b.Call(key, body) } func (b *fakeBus) CallFile(key string, body any) (json.RawMessage, error) { return b.Call(key, body) } +func (b *fakeBus) CallFileKept(_ process.KeptAsk, key string, body any) (json.RawMessage, error) { + return b.Call(key, body) +} func (b *fakeBus) Emit(eventType string, body any) error { b.events = append(b.events, fmt.Sprintf("%s %+v", eventType, body)) return nil diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go index b42fa8594..a3deaf66d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go @@ -21,6 +21,7 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-assigner/internal/assign" ) @@ -44,6 +45,8 @@ type caller interface { CallKept(key string, body any) (json.RawMessage, error) // CallFile is Call of a verb marked chunks, its answer read whole (process.Conn.CallFile; design 47 G A-291). CallFile(key string, body any) (json.RawMessage, error) + // CallFileKept is CallFile as a kept call carrying its ask's id, until its bound (process.Conn.CallFileKept; H2 §775). + CallFileKept(ask process.KeptAsk, key string, body any) (json.RawMessage, error) Emit(eventType string, body any) error } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/stepqueue.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/stepqueue.go new file mode 100644 index 000000000..0b9fdf309 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/stepqueue.go @@ -0,0 +1,318 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// **The seat mesh-wants takes its asks from a work queue and says ask-taken** (design 47 H2 §777, §775, §778; ADR 0525 §2, +// §6; ADR 0668 §2; H-A-381, H-A-401, H-A-404): the broker derives the seat's stream and its one shared worker from the seat's +// definition; the assigner pulls every ask waiting at once, takes each once by its id in its state taken (key +// ., for ask-taken-bound), says ask-taken at the pull, sends in-progress while the step runs and acknowledges an +// ask once it said the ask's end. An ask put while the assigner is down waits on the stream. The seat's act drop ends an ask +// its caller no longer waits for: one running here is cancelled, one not pulled yet is kept in taken as dropped. for +// ask-taken-bound, and either ends its step could not run naming the drop and who asked it. + +// askTakenBoundSetting is the setting the taken key's time to live is (H-A-381 (2)): the holder never receives the +// caller's bound, so the largest caller's, the runner's ask-unanswered-bound, is its default. +const ( + askTakenBoundSetting = "ask-taken-bound" + defaultAskTakenBound = 10 * time.Minute +) + +// droppedPrefix keys a drop of an ask not pulled yet in the state taken (H-A-404). +const droppedPrefix = "dropped." + +// takenStore is the state taken (process.State): an ask taken once by its id, and the drops of asks not pulled yet. +type takenStore interface { + TakeOnce(seat, ask string, bound time.Duration) (bool, error) + Get(key string) (*stdio.StateEntry, error) + RenewTTL(key string, value any, ttl time.Duration) (uint64, error) + Delete(key string) error +} + +// queueMsg is what the steps read of a message pulled from the seat's work queue (jetstream.Msg). +type queueMsg interface { + Data() []byte + Headers() nats.Header + Ack() error + Nak() error + Term() error + InProgress() error +} + +// askDrop is one drop of an ask: who asked it, why and when; the cause of a running step's end. +type askDrop struct { + By string `json:"by,omitempty"` + Why string `json:"why"` + At time.Time `json:"at"` +} + +func (d *askDrop) Error() string { + by := d.By + if by == "" { + by = "its asker" + } + return fmt.Sprintf("the ask was dropped by %s: %s", by, d.Why) +} + +// askIDOf is the ask's id a put carries: its event id, else its message id. +func askIDOf(h nats.Header) string { + if id := h.Get("x-event-id"); id != "" { + return id + } + return h.Get(nats.MsgIdHdr) +} + +// pulled handles one ask pulled from the seat's work queue: an ask that is no composition or validate step with an id is +// terminated, said; one dropped before it was pulled ends could not run naming the drop; one running here already (handed +// again while it runs) is held for that run, which acknowledges it at its end; any other is taken once by its id (ask-taken +// said the first time) and run, in progress while it runs. A repeat of an id taken before that runs here no longer is run +// again against the runs as they stand (H-A-381 (1), H-A-401): its composition runs are joined, never run twice. +func (c *compositionSteps) pulled(ctx context.Context, m queueMsg) { + id := askIDOf(m.Headers()) + var run meshwants.StepRun + err := json.Unmarshal(m.Data(), &run) + how := c.howOf(run.Step, false) + if err != nil || id == "" || how == nil { + c.logf("a step ask on the seat %s's work queue that is no composition or validate step with an id is ended: %v", meshwants.Seat, err) + _ = m.Term() + return + } + if d := c.droppedBefore(id); d != nil { + c.hold(id, m) + c.logf("the ask %s (%s) was dropped before it was pulled: %v", id, run.Pipeline, d) + c.sayEnd(id, run, nil, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: d.Error()}, "") + if err := c.taken.Delete(droppedPrefix + id); err != nil { + c.logf("the drop of the ask %s cannot be deleted from the state %s: %v", id, process.TakenState, err) + } + return + } + if c.running(id) { + c.hold(id, m) + c.logf("the ask %s (%s) was handed again while it runs here: its run answers it", id, run.Pipeline) + return + } + first, err := c.taken.TakeOnce(meshwants.Seat, id, c.takenBound) + if err != nil { + c.logf("the ask %s (%s) is not taken, the state %s refused it: %v; it is handed again", id, run.Pipeline, process.TakenState, err) + _ = m.Nak() + return + } + c.hold(id, m) + if first { + at := c.now().UTC() + if err := c.say(meshwants.EventAskTaken+"."+id, meshwants.AskTakenEvent{Ask: id, Node: c.node, At: at}, id+".taken"); err != nil { + c.logf("saying ask-taken of %s: %v", id, err) + } + } else { + c.logf("the ask %s (%s) was taken before: it is run again against its composition runs as they stand", id, run.Pipeline) + } + go c.say1(ctx, id, run, how, "") +} + +// hold keeps m for the step asked under id: acknowledged once its end is said, in progress while it runs. +func (c *compositionSteps) hold(id string, m queueMsg) { + c.mu.Lock() + defer c.mu.Unlock() + if c.held == nil { + c.held = map[string][]queueMsg{} + } + c.held[id] = append(c.held[id], m) +} + +// settle acknowledges (ack) or hands again (nak) every message held for id. +func (c *compositionSteps) settle(id string, ack bool) { + c.mu.Lock() + ms := c.held[id] + delete(c.held, id) + c.mu.Unlock() + for _, m := range ms { + if ack { + _ = m.Ack() + } else { + _ = m.Nak() + } + } +} + +// inProgressEvery says every message held for id in progress every d, until the returned stop is called. +func (c *compositionSteps) inProgressEvery(id string) (stop func()) { + every := time.Duration(0) + if c.inProgress != nil { + every = c.inProgress() + } + if every <= 0 { + return func() {} + } + ticks := c.ticks + if ticks == nil { + ticks = func(d time.Duration) (<-chan time.Time, func()) { t := time.NewTicker(d); return t.C, t.Stop } + } + tick, stopTicks := ticks(every) + done, ended := make(chan struct{}), make(chan struct{}) + go func() { + defer close(ended) + defer stopTicks() + for { + select { + case <-done: + return + case <-tick: + c.mu.Lock() + ms := append([]queueMsg(nil), c.held[id]...) + c.mu.Unlock() + for _, m := range ms { + _ = m.InProgress() + } + } + } + }() + return func() { close(done); <-ended } +} + +// droppedBefore is the drop kept for an ask not pulled yet, nil for none. +func (c *compositionSteps) droppedBefore(id string) *askDrop { + if c.taken == nil { + return nil + } + e, err := c.taken.Get(droppedPrefix + id) + if err != nil { + c.logf("whether the ask %s was dropped cannot be read from the state %s: %v; it is taken", id, process.TakenState, err) + return nil + } + if e == nil { + return nil + } + var d askDrop + if json.Unmarshal(e.Value, &d) != nil { + return nil + } + return &d +} + +// dropAsk is the seat's act drop (§778, H-A-404): a step running here for the ask is cancelled with the drop as its cause and +// ends could not run naming it; one not running here is kept in taken as dropped. for ask-taken-bound, so its pull ends +// it the same way. It answers whether the ask was held here. +func (c *compositionSteps) dropAsk(caller string, in meshwants.DropInput) (meshwants.DropAnswer, error) { + id, why := strings.TrimSpace(in.ID), strings.TrimSpace(in.Why) + if id == "" || why == "" { + return nil, errors.New("give the ask's id and why it is dropped") + } + by := strings.TrimSpace(in.By) + if by == "" { + by = caller + } + d := &askDrop{By: by, Why: why, At: c.now().UTC()} + c.mu.Lock() + stop := c.stops[id] + c.mu.Unlock() + if stop != nil { + stop(d) + c.logf("the ask %s running here is dropped: %v", id, d) + return meshwants.DropAnswer{"held": true}, nil + } + if c.taken == nil { + return nil, fmt.Errorf("the state %s is not bound, so the drop of %s cannot be kept", process.TakenState, id) + } + if _, err := c.taken.RenewTTL(droppedPrefix+id, d, c.takenBound); err != nil { + return nil, fmt.Errorf("writing the drop of %s in the state %s: %w", id, process.TakenState, err) + } + c.logf("the ask %s, not running here, is kept dropped until it is pulled: %v", id, d) + return meshwants.DropAnswer{"held": false}, nil +} + +// dropVerb is the process's handler of the seat mesh-wants' act drop. +func dropVerb(c *compositionSteps) map[string]process.SeatHandler { + return map[string]process.SeatHandler{ + meshwants.VerbDrop: func(caller string, raw json.RawMessage) (any, error) { + var in meshwants.DropInput + if err := json.Unmarshal(raw, &in); err != nil { + return nil, fmt.Errorf("the arguments are not JSON: %w", err) + } + return c.dropAsk(caller, in) + }, + } +} + +// inProgressOf is the seat's in-progress as its definition names it, once the seat list is read; before, a third of the +// worker's ack wait, which the broker derived from the same definition (60 s and 20 s, §777; H-A-401). +func inProgressOf(def func() (seatdef.Definition, bool), ackWait func() time.Duration) func() time.Duration { + return func() time.Duration { + if d, ok := def(); ok { + if _, worker, ok, err := buscompose.SeatQueue(meshwants.Seat, d.Accepts); err == nil && ok && worker.InProgressSeconds > 0 { + return time.Duration(worker.InProgressSeconds) * time.Second + } + } + return ackWait() / 3 + } +} + +// serveStepQueue pulls the seat's work queue through its one shared worker, every ask waiting at once (at most the worker's +// max ack pending), once the broker made it: a worker not there yet is said, and bound when the bus says it was created. +func serveStepQueue(ctx context.Context, conn process.Subscriber, js jetstream.JetStream, c *compositionSteps, ackWait func(time.Duration)) error { + stream, worker := buscompose.SeatQueueStreamName(meshwants.Seat), buscompose.SeatQueueWorkerName(meshwants.Seat) + var once sync.Once + bind := func() bool { + cons, err := js.Consumer(ctx, stream, worker) + if err != nil { + return false + } + bound := false + once.Do(func() { + bound = true + info := cons.CachedInfo() + if info != nil && info.Config.AckWait > 0 { + ackWait(info.Config.AckWait) + } + pending := 1000 + if info != nil && info.Config.MaxAckPending > 0 { + pending = info.Config.MaxAckPending + } + cc, err := cons.Consume(func(m jetstream.Msg) { c.pulled(ctx, m) }, jetstream.PullMaxMessages(pending)) + if err != nil { + c.logf("the seat %s's work queue %s is not pulled: %v", meshwants.Seat, worker, err) + return + } + c.logf("the composition step is pulled from %s on %s", worker, stream) + go func() { <-ctx.Done(); cc.Stop() }() + }) + return bound + } + if bind() { + return nil + } + c.logf("the seat %s's work queue %s on %s is not there yet: its asks wait on it, and it is pulled when the bus says it was created", meshwants.Seat, worker, stream) + sub, err := conn.Subscribe("$JS.EVENT.ADVISORY.CONSUMER.CREATED."+stream+"."+worker, func(*nats.Msg) { go bind() }) + if err != nil { + return fmt.Errorf("hearing the creation of the seat %s's work queue: %w", meshwants.Seat, err) + } + go func() { <-ctx.Done(); _ = sub.Unsubscribe() }() + if bind() { // made between the first look and the subscription + _ = sub.Unsubscribe() + } + return nil +} + +// askTakenBound reads the setting ask-taken-bound (§781). +func askTakenBound() (time.Duration, error) { + d, err := stdio.SettingDuration(askTakenBoundSetting, defaultAskTakenBound) + if err != nil { + return 0, errors.New("the assigner's " + err.Error()) + } + return d, nil +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go index c71e7dbd5..eb704a538 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go @@ -22,6 +22,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/seatlist" "git.novox.be/novox/mesh-sdk/go/stateobjects" "mesh-delivery-assigner/internal/buckets" @@ -416,6 +417,9 @@ func serveTools() error { tools = append(tools, whyTools(runtimeStore{})...) tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...) tools = append(tools, wantDropTools(runtimeStore{}, stdio.Ask)...) + tools = append(tools, asksTools(stdio.QueueWaiting, stdio.State(process.TakenState), stdio.Ask)...) // the seat's work queue (§777) + tools = append(tools, keepNowTools(stdio.Ask)...) + tools = append(tools, validateBeforeTools(runtimeStore{})...) // what a validate step did not count (§814) // the keeping by hand (§813 (2)) return stdio.Serve("", tools) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep.go index 7d3bacc8f..c513565f1 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep.go @@ -2,6 +2,10 @@ package main import ( "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -15,6 +19,10 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-assigner/internal/buckets" ) // The step validate (design 47 H §745; H-A-34, H-A-51, H-A-108): the assigner serves it as an accepted kind `step` of the seat @@ -23,8 +31,8 @@ import ( // reads: the cycle the revision's entries make (graph.RefusedCycle) ends it at once; else the manifest validator built from // the revision's own entry of the module holding the seat mesh-catalog, on the Go build seat as the composer is, runs in the // sandbox over the revision's manifests and its base's (H-A-154), its input kept and served on mesh-wants' input. -// Every mover's step ends the same: failed `cycle` naming the cycle, failed `manifest-refused` naming every problem of a -// manifest the revision alters or between manifests one it alters takes part in, or passed. +// Every mover's step ends the same: failed `cycle` naming the cycle, failed `manifest-refused` naming every problem the case +// before does not hold (design 47 H2 §814; assigner_validate_before answers the ones it holds), or passed. const ( // validatorPackage is the manifest validator's import path in the module holding the seat mesh-catalog (the indexer's @@ -112,16 +120,12 @@ func (r *compositionRun) judgeValidate(ctx context.Context, step meshwants.StepR if err != nil { return couldNotRun("the step's catalog revision is not read: " + err.Error()) } - before, err := r.before(ctx, step, half) - if err != nil { - return couldNotRun("the base of the case before is not read: " + err.Error()) - } // A revision that names no base (the branch's first take-in) is judged against an empty case before, and says so (H-A-151). empty := "" if half.Revision.Base <= 0 { empty = emptyBeforeWords } - if problems := causedProblems(v.Change, v.Before, altered(half, before)); len(problems) > 0 { + if problems := causedProblems(v.Change, v.Before); len(problems) > 0 { said := reasonManifestRefused + ": " + strings.Join(problems, "; ") if empty != "" { said += " (" + empty + ")" @@ -173,60 +177,44 @@ func (r *compositionRun) before(ctx context.Context, step meshwants.StepRun, hal return r.revision(ctx, base) } -// altered are the modules whose manifest the revision adds or changes against the one before it. -func altered(half, before mcc.Half) []string { - var out []string - for name, e := range half.Entries { - if len(e.Manifest) == 0 { - continue - } - b, ok := before.Entries[name] - switch { - case !ok || len(b.Manifest) == 0: - out = append(out, name) - case e.ManifestDigest != "" && b.ManifestDigest != "": - if e.ManifestDigest != b.ManifestDigest { - out = append(out, name) - } - default: - x, _ := json.Marshal(e.Manifest) - y, _ := json.Marshal(b.Manifest) - if string(x) != string(y) { - out = append(out, name) - } - } - } - slices.Sort(out) - return out +// causedProblems are the problems the revision causes (ADR 0578 §2; design 47 H2 §814): of every manifest, and between +// manifests, each problem the case before did not have; a problem the case before holds is never counted, whether the +// revision alters that manifest or not (assigner_validate_before answers those). +func causedProblems(change, before composer.CaseVerdict) []string { + return problemsBy(change, before, false) } -// causedProblems are the problems the revision causes (ADR 0578 §2): every problem of a manifest it alters, and of any other -// manifest, or between manifests, each problem the case before did not have. -func causedProblems(change, before composer.CaseVerdict, touched []string) []string { +// uncountedProblems are the change case's problems the case before holds too, which causedProblems does not count (§814). +func uncountedProblems(change, before composer.CaseVerdict) []string { + return problemsBy(change, before, true) +} + +// problemsBy are the change case's problems, per manifest, that the case before had (had) or did not have (!had). +func problemsBy(change, before composer.CaseVerdict, had bool) []string { var out []string for _, module := range slices.Sorted(maps.Keys(change.Manifests)) { read := change.Manifests[module] if read.Accepted || read.Refused == "" { continue } - had := map[string]bool{} + held := map[string]bool{} for _, p := range strings.Split(before.Manifests[module].Refused, "\n") { - had[p] = true + held[p] = true } - var caused []string + var named []string for _, p := range strings.Split(read.Refused, "\n") { - if p != "" && (slices.Contains(touched, module) || !had[p]) { - caused = append(caused, p) + if p != "" && held[p] == had { + named = append(named, p) } } - if len(caused) == 0 { + if len(named) == 0 { continue } what := module if module == betweenManifests { what = "between manifests" } - out = append(out, what+": "+strings.Join(caused, "; ")) + out = append(out, what+": "+strings.Join(named, "; ")) } return out } @@ -300,7 +288,7 @@ func (r *compositionRun) startValidate(ctx context.Context, key string, run mesh return } program.Package = validatorPackage - seats, err := r.seats(ctx, half.Revision.Definitions) + seats, list, err := r.validateSeats(ctx, ask, half.Revision.Definitions) if err != nil { couldNot(err.Error()) return @@ -317,7 +305,7 @@ func (r *compositionRun) startValidate(ctx context.Context, key string, run mesh } // No predefined key list and no known package names until a design line names the verb that gives them (H-A-108 (2)). req := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: changeCase, Manifests: change}, {Name: beforeCase, Manifests: prior}}, - Seats: seats} + Seats: seats, SeatList: list} raw, err := json.Marshal(req) if err != nil { couldNot("its request cannot be encoded: " + err.Error()) @@ -380,3 +368,126 @@ func validateRunVerdict(answer []byte) (map[string]any, string) { } return verdict, "" } + +// validateBeforeTool is the tool that answers what a validate step did not count (design 47 H2 §814). +const validateBeforeTool = "assigner_validate_before" + +// validateBefore answers, for the validate run of pipeline, the problems of the change case that its case before holds too, +// which the step did not count, for module alone when one is named; read from the run the state composition-runs keeps. +func validateBefore(st store, pipeline, module string) (map[string]any, error) { + key, err := validateKeyOf(meshwants.StepRun{Pipeline: strings.TrimSpace(pipeline)}) + if err != nil { + return nil, err + } + raw, _, ok, err := st.get(buckets.CompositionRunsState, key) + if err != nil { + return nil, fmt.Errorf("the state %s is not read: %w", buckets.CompositionRunsState, err) + } + if !ok { + return nil, fmt.Errorf("no validate run %s is kept", key) + } + var run meshwants.CompositionRun + if err := json.Unmarshal(raw, &run); err != nil { + return nil, fmt.Errorf("the validate run %s does not read: %w", key, err) + } + var v validateVerdict + if b, err := json.Marshal(run.Verdict); err != nil || json.Unmarshal(b, &v) != nil { + return nil, fmt.Errorf("the validate run %s holds no verdict that reads", key) + } + change, before := v.Change, v.Before + if module = strings.TrimSpace(module); module != "" { + change = composer.CaseVerdict{Name: change.Name, Manifests: map[string]composer.ManifestRead{module: change.Manifests[module]}} + } + uncounted := uncountedProblems(change, before) + if uncounted == nil { + uncounted = []string{} + } + return map[string]any{"run": key, "outcome": run.Outcome, "uncounted": uncounted}, nil +} + +// validateBeforeTools are the module's tool of the validate step's before case. +func validateBeforeTools(st store) []stdio.Tool { + return []stdio.Tool{{Name: validateBeforeTool, Description: "The problems a validate step did not count (design 47 H2 §814): of the " + + "validate run of a pipeline's take-in revision, every problem of the change case its case before holds too, per manifest, " + + "for one module when named. Read-only.", + Input: schema(map[string]string{"pipeline": "the pipeline's key, .", "module": "one module; absent: every one"}, []string{"pipeline"}), + Run: func(args map[string]any) (any, error) { + return validateBefore(st, arg(args, "pipeline"), arg(args, "module")) + }}} +} + +// seatListKey is the seat mesh-catalog's chunked read verb seat-list (design 47 H2 §814, H-A-441, H-A-443). +const seatListKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbSeatList + +// validateSeats are the seat definitions a validate run judges by, as one JSON array in composer.Request.Seats' form sorted by +// seat, and their digest, the sha256 of those bytes (H-A-441, H-A-443): a revision that names its own definitions keeps them, +// each read through seat-definitions; one that names none takes the trunk's seat list, the bytes the seat mesh-catalog's verb +// seat-list hands, asked as a kept call under the run's ask. +func (r *compositionRun) validateSeats(ctx context.Context, ask string, defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, string, error) { + if len(defs) == 0 { + return r.seatList(ask) + } + sorted := slices.Clone(defs) + slices.SortStableFunc(sorted, func(a, b meshcatalog.CatalogRevisionDefinitionsItem) int { return strings.Compare(a.Seat, b.Seat) }) + seats, err := r.seats(ctx, sorted) + if err != nil { + return nil, "", err + } + raw, err := json.Marshal(seats) + if err != nil { + return nil, "", fmt.Errorf("the revision's seat definitions cannot be encoded: %w", err) + } + var list []json.RawMessage + if err := json.Unmarshal(raw, &list); err != nil { + return nil, "", fmt.Errorf("the revision's seat definitions do not read again: %w", err) + } + return list, seatListDigest(raw), nil +} + +// seatListDigest is the digest of a seat list's bytes, as the indexer names its object in seat-lists: the sha256, hex. +func seatListDigest(raw []byte) string { + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]) +} + +// seatList is the trunk's seat list as the verb seat-list hands it (H-A-443): its digest and size, then the object's bytes in +// chunks; bytes whose size or sha256 is not the answer's, or that are no JSON array, are refused, so every run of one digest +// judges the same bytes. +func (r *compositionRun) seatList(ask string) ([]json.RawMessage, string, error) { + what := "the trunk's seat list is not read through " + seatListKey + if r.callFile == nil { + return nil, "", errors.New(what + ": the assigner holds no chunked call") + } + var bits [16]byte + if _, err := rand.Read(bits[:]); err != nil { + return nil, "", fmt.Errorf("%s: %w", what, err) + } + raw, err := r.callFile(process.KeptAsk{ID: ask + "." + meshcatalog.VerbSeatList, Bound: readsBound}, seatListKey, + meshcatalog.SeatListInput{Call: hex.EncodeToString(bits[:])}) + if err != nil { + return nil, "", fmt.Errorf("%s: %w", what, err) + } + var a struct { + meshcatalog.SeatListAnswer + File string `json:"file"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, "", fmt.Errorf("%s: its answer does not read: %w", what, err) + } + if a.Digest == "" { + return nil, "", errors.New(what + ": its answer names no digest") + } + file, err := base64.StdEncoding.DecodeString(a.File) + if err != nil { + return nil, "", fmt.Errorf("%s: the bytes received are not base64: %w", what, err) + } + if int64(len(file)) != a.Size || seatListDigest(file) != a.Digest { + return nil, "", fmt.Errorf("%s: %d bytes of digest %s were received, the answer names %d bytes of digest %s", what, len(file), + seatListDigest(file), a.Size, a.Digest) + } + var list []json.RawMessage + if err := json.Unmarshal(file, &list); err != nil || len(list) == 0 { + return nil, "", fmt.Errorf("%s: the seat list %s is no JSON array of seat definitions", what, a.Digest) + } + return list, a.Digest, nil +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go index 0326f4f19..cbe180e69 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go @@ -2,8 +2,10 @@ package main import ( "context" + "encoding/base64" "encoding/json" "errors" + "slices" "strings" "sync" "testing" @@ -14,6 +16,9 @@ import ( meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-assigner/internal/buckets" ) // A10/AC1, A10/AC2 (design 47 H §745; H-A-34, H-A-51, H-A-108): every mover's validate step of one take-in reads one validate @@ -60,19 +65,40 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) { if !ok || in.Branch != "main" { return nil, errors.New("asked another revision") } + // Revision 3 names no definitions of its own: it is judged by the trunk's seat list, the bytes seat-list hands (H-A-443 (2)). + var defs []meshcatalog.CatalogRevisionDefinitionsItem + if in.Revision != 3 { + defs = []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "sd"}} + } return json.Marshal(meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: in.Revision, Base: base(in.Revision), - Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "sd"}}}, Entries: entries}) + Definitions: defs}, Entries: entries}) case seatDefinitionsKey: return json.Marshal(meshcatalog.SeatDefinitionsAnswer{Found: true, Entry: &meshcatalog.SeatDefinitionsAnswerEntry{Seat: "s1", Digest: "sd", Definition: map[string]any{"seat": "s1"}}}) } return nil, errors.New("no such verb " + key) } + // The trunk's seat list as the verb seat-list hands it (H-A-443 (1)): digest and size, then the bytes, which CallFileKept adds as file. + trunkList := []byte(`[{"seat":"a-seat"},{"seat":"s1","v":2}]`) + listAnswer := func(file []byte, digest string, size int) json.RawMessage { + raw, _ := json.Marshal(map[string]any{"digest": digest, "size": size, "chunks": 1, "file": base64.StdEncoding.EncodeToString(file)}) + return raw + } + var listAsks []process.KeptAsk + listed := listAnswer(trunkList, seatListDigest(trunkList), len(trunkList)) + callFile := func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) { + in, _ := body.(meshcatalog.SeatListInput) + if key != seatListKey || len(in.Call) != 32 { + return nil, errors.New("asked " + key + " with the call " + in.Call) + } + listAsks = append(listAsks, ask) + return listed, nil + } var mu sync.Mutex var asks []buildstep.StepAsk var requests []composer.Request keeper := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now) runs := newCompositionRuns(newRunsMem(), defaultRunSettings, time.Now, t.Logf) - r := &compositionRun{runs: runs, keeper: keeper, reads: &compositionReads{}, call: call, now: time.Now, logf: t.Logf, + r := &compositionRun{runs: runs, keeper: keeper, reads: &compositionReads{}, call: call, callFile: callFile, now: time.Now, logf: t.Logf, put: func(_ context.Context, a buildstep.StepAsk) error { body, err := keeper.body(a.Run.Input.Digest) if err != nil { @@ -86,8 +112,8 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) { asks, requests = append(asks, a), append(requests, req) mu.Unlock() answer, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{ - {Name: changeCase, Manifests: map[string]composer.ManifestRead{"web": {Refused: `unknown field "colour"`}, "db": {Refused: "an old problem"}, "indexer": {Accepted: true}}}, - {Name: beforeCase, Manifests: map[string]composer.ManifestRead{"db": {Refused: "an old problem"}}}}}) + {Name: changeCase, Manifests: map[string]composer.ManifestRead{"web": {Refused: "unknown field \"colour\"\nan inherited problem"}, "db": {Refused: "an old problem"}, "indexer": {Accepted: true}}}, + {Name: beforeCase, Manifests: map[string]composer.ManifestRead{"db": {Refused: "an old problem"}, "web": {Refused: "an inherited problem"}}}}}) go func() { if err := runs.heardEnd(buildstep.StepEnded{Asker: runAsker, Ask: a.ID, Outcome: buildstep.OutcomePassed, Node: "novox"}, func(buildstep.StepEnded) ([]byte, error) { return answer, nil }); err != nil { @@ -115,10 +141,21 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) { wg.Wait() for i, o := range outs { if o.Outcome != meshwants.StepEndedOutcomeFailed || !strings.HasPrefix(o.Said, reasonManifestRefused+": ") || !strings.Contains(o.Said, `web: unknown field "colour"`) || - strings.Contains(o.Said, "an old problem") { + strings.Contains(o.Said, "an old problem") || strings.Contains(o.Said, "an inherited problem") { // H2 §814: only new problems t.Errorf("mover %d's validate step ended %+v", i, o) } } + // assigner_validate_before (H2 §814): the problems the before case holds, which the step did not count. + runsKV := memStore{buckets.CompositionRunsState: &memKV{values: map[string][]byte{}}} + kept, _, _, _ := runs.read("main.7" + validateKeySuffix) + runsKV[buckets.CompositionRunsState].values["main.7"+validateKeySuffix], _ = json.Marshal(kept) + before, err := validateBeforeTools(runsKV)[0].Run(map[string]any{"pipeline": "main.7", "module": "web"}) + if b, _ := before.(map[string]any); err != nil || !slices.Equal(b["uncounted"].([]string), []string{"web: an inherited problem"}) { + t.Errorf("assigner_validate_before answered %+v, %v", before, err) + } + if _, err := validateBeforeTools(runsKV)[0].Run(map[string]any{"pipeline": "main.6"}); err == nil { + t.Error("assigner_validate_before answers a run not kept") + } if len(asks) != 1 { t.Fatalf("%d validator asks were put for one take-in, want 1", len(asks)) } @@ -132,6 +169,11 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) { if c := requests[0].Cases; len(c) != 2 || c[0].Name != changeCase || c[1].Name != beforeCase || len(c[1].Manifests) != 3 || requests[0].Predefined != nil || requests[0].Packages != nil { t.Errorf("the validator's request is %+v", requests[0]) } + // A revision naming its own definitions keeps them, and the request names the sha256 of their bytes (H-A-443 (2)); no seat + // list is asked. + if own := []byte(`[{"seat":"s1"}]`); requests[0].SeatList != seatListDigest(own) || len(requests[0].Seats) != 1 || string(requests[0].Seats[0]) != `{"seat":"s1"}` || len(listAsks) != 0 { + t.Errorf("a revision naming its definitions was judged by %s %s (%d seat-list asks), want %s", requests[0].SeatList, requests[0].Seats, len(listAsks), seatListDigest(own)) + } if _, _, ok, _ := runs.read("main.7" + validateKeySuffix); !ok { t.Error("the validate run is not the key .validate of composition-runs") } @@ -144,6 +186,29 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) { o.Said != reasonManifestRefused+`: web: unknown field "colour" (`+emptyBeforeWords+")" || len(asks) != 2 || len(requests[1].Cases[1].Manifests) != 0 { t.Errorf("a revision naming no base ended %+v (%d validator asks), want it judged against an empty case before, said", o, len(asks)) } + // Revision 3 names no definitions: Seats are the trunk's seat list's bytes as seat-list handed them, named by its digest, + // asked once as a kept call under the run's ask (H-A-441 (b), H-A-443). + if got, _ := json.Marshal(requests[1].Seats); requests[1].SeatList != seatListDigest(trunkList) || string(got) != string(trunkList) || + len(listAsks) != 1 || listAsks[0].ID != asks[1].ID+"."+meshcatalog.VerbSeatList || listAsks[0].Bound <= 0 { + t.Errorf("a revision naming no definitions was judged by %s %s (asks %+v), want the trunk's seat list %s", requests[1].SeatList, got, listAsks, seatListDigest(trunkList)) + } + // Bytes the answer does not name, by size or by digest, or that are no seat list, are refused: the run judges no other bytes. + for _, row := range []struct { + name string + answer json.RawMessage + }{ + {"a digest of other bytes", listAnswer(trunkList, seatListDigest([]byte("[]")), len(trunkList))}, + {"another size", listAnswer(trunkList, seatListDigest(trunkList), len(trunkList)+1)}, + {"no digest", listAnswer(trunkList, "", len(trunkList))}, + {"no JSON array", listAnswer([]byte(`{"seat":"s1"}`), seatListDigest([]byte(`{"seat":"s1"}`)), len(`{"seat":"s1"}`))}, + {"an empty list", listAnswer([]byte(`[]`), seatListDigest([]byte(`[]`)), 2)}, + } { + listed = row.answer + if seats, digest, err := r.seatList("ask-row"); err == nil || !strings.Contains(err.Error(), seatListKey) { + t.Errorf("%s: the seat list read %s %s, %v; want it refused, naming the verb", row.name, digest, seats, err) + } + } + listed = listAnswer(trunkList, seatListDigest(trunkList), len(trunkList)) // Revision 12 holds revision 7's manifests and names base 7: judged against it, web's problem is the revision's, db's old one // is not, and the case before holds base 7's manifests. if o := r.validate(ctx, "ask-base", step("web", 12)); o.Outcome != meshwants.StepEndedOutcomeFailed || diff --git a/modules/mesh-delivery-assigner/internal/buckets/buckets.go b/modules/mesh-delivery-assigner/internal/buckets/buckets.go index a894baafb..a3aacb739 100644 --- a/modules/mesh-delivery-assigner/internal/buckets/buckets.go +++ b/modules/mesh-delivery-assigner/internal/buckets/buckets.go @@ -44,6 +44,10 @@ func Inputs() string { return own(InputsState) } func CompositionRuns() string { return own(CompositionRunsState) } func Policies() string { return own(PoliciesState) } +// Taken is the bucket of the state taken (design 47 H2 §775, §777): the asks of the seat mesh-wants taken once, and their +// drops; named as mesh-sdk's process.TakenState. +func Taken() string { return own("taken") } + // AsksHistory is the revisions the bucket of asks keeps per key. const AsksHistory = 5 diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 8a3db0c22..ab95812aa 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -70,7 +70,7 @@ func TestManifest(t *testing.T) { } if want := map[string]int{"asks": 5, "differences": 64, "puts": 20, "wants": 20, "assignments": 64, "placement": 64, "copies": 1, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5, views.MarksState: 1, views.StatusState: 1, - "engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "inputs": 1, "composition-runs": 1}; !reflect.DeepEqual(states, want) { + "engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "inputs": 1, "composition-runs": 1, "taken": 1}; !reflect.DeepEqual(states, want) { t.Errorf("state = %v, want %v", states, want) } if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 { @@ -93,7 +93,7 @@ func TestManifest(t *testing.T) { // revision, entry and difference from its revisions kept as objects in the old section, form 1, capped at 512 MiB. if len(m.Claims) != 2 || m.Claims[0].Name != meshmoduleassigner.Seat || m.Claims[0].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[0].Serves, []string{"assignments", "proposals", "propose", "withdraw", "again", "placement"}) || - m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input", "compositions", "compositions-drop"}) { + m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input", "compositions", "compositions-drop", "keep-now", "drop"}) { t.Errorf("the assigner claims %+v, want mesh-module-assigner with its five verbs and mesh-wants with its five", m.Claims) } if len(m.Objects) != 1 || m.Objects[0].Name != "assignment-revisions" || m.Objects[0].MaxMiB != 512 || m.Objects[0].Form != 1 { @@ -105,7 +105,7 @@ func TestManifest(t *testing.T) { // which lists and asks again what each node's engine reads. if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_waits", "assigner_inputs", "assigner_inputs_drop", "assigner_compositions", "assigner_compositions_drop", "assignments", "seat-holders", "wants", "asks", "assigner_read_assignments", "assigner_read_seat_holders", "assigner_read_wants", "assigner_read_findings", "assigner_read_differences", "assigner_read_copies", - "assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_want_drop"}) { + "assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_want_drop", "assigner_asks", "assigner_ask_drop", "assigner_keep_now", "assigner_validate_before"}) { t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads, the seat list's two (ADR 0636 §7) assigner_ask_again (ADR 0668 §5) and assigner_why (design 47 G A-439)", m.Tools) } if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs { @@ -148,13 +148,14 @@ func TestManifest(t *testing.T) { if want := []string{"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", buckets.TellerModule + "." + buckets.AnswersReadTool, // A-175, A-178: the teller's answers' first view "seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list + "seat:mesh-catalog.seat-list", // design 47 H2 H-A-443: the validate step's trunk seat list, in chunks "seat:mesh-conditions.conditions", // design 47 G §706: the conditions read at each judging of an outside wait "seat:mesh-delivery.source", // §507: the branches the gate keeps a source revision of, before the first registrations "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone "seat:node-backup.backed-up", "seat:node-backup.now", "seat:node-build-go.accept.check", // A-231 + "seat:node-build-go.answer", // A-254, H-A-412: a composition run's answer over the inline bound, read through the build seat's verb answer "seat:node-build-go.drop", // §720, A-469s: a composition run past its bound drops its ask: the composition run's ask on the Go build seat's queue check (accept.check.) - "seat:node-build-go.output", // A-254: a composition run's answer kept as the build seat's object "seat:node-engine.compose-candidate", // §354, A-226: each machine's composition input "seat:operator-channel.asks"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) { // ADR 0668 §4: the ended asks at bind t.Errorf("invokes = %v, want %v", m.Invokes, want) @@ -172,9 +173,11 @@ func TestManifest(t *testing.T) { } file, _ := os.ReadFile("module.json") if err := json.Unmarshal(file, &raw); err != nil || raw.Config["outside-wait-minutes"].Default != float64(30) || raw.Config["outside-settle-seconds"].Default != float64(90) || - raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 10 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) || + raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 15 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) || raw.Config["composition-runs-cap-mib"].Default != float64(16) || raw.Config["composition-runs-kept"].Default != "24h" || - raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) { + raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) || + // design 47 H2 §775, H-A-381 (2): the taken key's bound and the kept-call pace, runtime settings + raw.Config["ask-taken-bound"].Default != "10m" || raw.Config["kept-call-pace-first"].Default != "1s" || raw.Config["kept-call-pace-max"].Default != "30s" || raw.Config["compose-ask-bound"].Default != "10s" || raw.Config["keeping-retry"].Default != "30s" { t.Errorf("config = %+v (%v), want outside-wait-minutes of 30, outside-settle-seconds of 90 and ask-operator-for of take-off,remove-node,stops (A-425a)", raw.Config, err) } for _, f := range []struct{ env, path, key string }{ diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 329791381..b6a11c4b4 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -27,7 +27,9 @@ "difference", "input", "compositions", - "compositions-drop" + "compositions-drop", + "keep-now", + "drop" ] } ], @@ -70,6 +72,7 @@ "seat:mesh-catalog.registrations", {"verb": "seat:mesh-catalog.revision", "fields": ["none-yet"]}, "seat:mesh-catalog.seat-definitions", + "seat:mesh-catalog.seat-list", "seat:mesh-conditions.conditions", "seat:mesh-delivery.source", "seat:mesh-hand-acts.hand-act", @@ -77,7 +80,7 @@ "seat:mesh-hand-acts.warranted", "seat:node-backup.backed-up", "seat:node-backup.now", - "seat:node-build-go.output", + "seat:node-build-go.answer", "seat:node-build-go.drop", "seat:node-engine.compose-candidate", "seat:operator-channel.asks", @@ -129,7 +132,11 @@ "assigner_ask_again", "assigner_written", "assigner_why", - "assigner_want_drop" + "assigner_want_drop", + "assigner_asks", + "assigner_ask_drop", + "assigner_keep_now", + "assigner_validate_before" ], "state": [ { "name": "assignments", "history": 64 }, @@ -152,7 +159,8 @@ { "name": "waits", "history": 5 }, { "name": "put-waits", "history": 1 }, { "name": "inputs", "history": 1, "max-mib": 64 }, - { "name": "composition-runs", "history": 1, "key-ttl": true, "max-mib": 64 } + { "name": "composition-runs", "history": 1, "key-ttl": true, "max-mib": 64 }, + { "name": "taken", "history": 1, "key-ttl": true, "max-mib": 8 } ], "config": { "outside-wait-minutes": { @@ -226,6 +234,41 @@ "maximum": 1000, "trusted": false, "default": 20 + }, + "ask-taken-bound": { + "type": "string", + "description": "how long the assigner keeps the id of an ask it took from the seat mesh-wants' work queue, so a repeat of it is taken once, and the drop of an ask not pulled yet, a Go duration such as 10m (design 47 H2 §775, §778; H-A-381 (2), H-A-404).", + "trusted": false, + "default": "10m", + "runtime": true + }, + "compose-ask-bound": { + "type": "string", + "description": "how long the assigner asks a machine's node-engine compose-candidate again through a transport fault before the machine gave no input, a Go duration such as 10s (design 47 H2 §775).", + "trusted": false, + "default": "10s", + "runtime": true + }, + "keeping-retry": { + "type": "string", + "description": "how long after a failed keeping of the assignment graph revisions the assigner keeps them again; keeping-failed is raised at the third failure in a row, a Go duration such as 30s (design 47 H2 §813 (2)).", + "trusted": false, + "default": "30s", + "runtime": true + }, + "kept-call-pace-first": { + "type": "string", + "description": "the first wait before a kept call met by a transport fault is made again, doubled per call up to kept-call-pace-max, a Go duration (design 47 H2 §775).", + "trusted": false, + "default": "1s", + "runtime": true + }, + "kept-call-pace-max": { + "type": "string", + "description": "the most a kept call's wait between calls grows to, a Go duration (design 47 H2 §775).", + "trusted": false, + "default": "30s", + "runtime": true } }, "objects": [ diff --git a/modules/mesh-delivery-contracts/go/seats/meshdelivery/contract.go b/modules/mesh-delivery-contracts/go/seats/meshdelivery/contract.go index 93579d021..5ee0a165e 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshdelivery/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshdelivery/contract.go @@ -20,7 +20,7 @@ const Form = 2 // whose fields changed while its forms did not (design 47 H2 §780). const ( generatedForms = "2" - generatedFieldsDigest = "DeliveriesAnswer=d6ae3ca4e5da Fill=80f442916429 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 HeadsByRepositoryBranchValue=b32fd594ee25 Owed=005330be954f OwedByRepositoryNumberHeadActValue=f16ff5733d62 OwedEvent=86af3b56de28 RecordEndedEvent=566edb8684e5 RecordEndedEventAwayItem=44ad7d26bd16 RecordOpenedEvent=9cf9afaa25b4 RecoveryAnswer=8c705e17cdf8 RecoveryMergedEvent=014d4d00d2e5 RoomEvent=5bd4df78ed84 ShowAnswer=7538f2f5e68e SourceAnswer=6ae84f05f10d SourceByBranchValue=31aa822718c0 SourceHead=e722e2d9492f SourceMovedEvent=5231909928e9 SourceMovedEventHeadsItem=774b4bf88ba4 StalledAnswer=7538f2f5e68e" + generatedFieldsDigest = "DeliveriesAnswer=d6ae3ca4e5da Fill=80f442916429 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 HeadsByRepositoryBranchValue=b32fd594ee25 Owed=a30ee01924cc OwedByRepositoryNumberHeadActValue=f16ff5733d62 OwedEvent=86af3b56de28 RecordEndedEvent=566edb8684e5 RecordEndedEventAwayItem=44ad7d26bd16 RecordOpenedEvent=9cf9afaa25b4 RecoveryAnswer=8c705e17cdf8 RecoveryMergedEvent=014d4d00d2e5 RoomEvent=5bd4df78ed84 ShowAnswer=7538f2f5e68e SourceAnswer=6ae84f05f10d SourceByBranchValue=31aa822718c0 SourceHead=e722e2d9492f SourceMovedEvent=5231909928e9 SourceMovedEventHeadsItem=774b4bf88ba4 StalledAnswer=7538f2f5e68e" ) const ( @@ -262,6 +262,8 @@ type Owed struct { Line string `json:"line,omitempty"` // the pull request's number Number int64 `json:"number"` + // the delivery record the act is for, by its source revision, .; absent on an act said before it was named, which bridge_owed_again by record never finds (design 47 H2 §807, H-A-432) + Record string `json:"record,omitempty"` // the repository, / Repository string `json:"repository"` // when the gate first said it diff --git a/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go b/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go index e661349ab..8f93ec1c9 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go @@ -20,7 +20,7 @@ const Form = 2 // whose fields changed while its forms did not (design 47 H2 §780). const ( generatedForms = "2" - generatedFieldsDigest = "Ask=6cb8b82e2fbe AskTakenEvent=6d5d3da7ac39 AsksByAssignmentsProposalValue=0bb5682c7b3b AsksByBackupNodeModuleFingerprintValue=0bb5682c7b3b AsksByHoldModuleFingerprintValue=0bb5682c7b3b AsksByOutsideNodeModuleFingerprintValue=0bb5682c7b3b AsksByRecordNodeModuleDigestValue=0bb5682c7b3b AsksByRestoreNodeModuleFingerprintValue=0bb5682c7b3b AsksByRevertNodeModuleFingerprintValue=0bb5682c7b3b AssignmentEntry=7d2549782eb2 AssignmentRevision=72c8701f7314 AssignmentRevisionCapabilitiesItem=0c89b0717762 AssignmentRevisionEvent=7faff968ce44 AssignmentRevisionsByCandidateGroupHeadsValue=8553b390118d AssignmentRevisionsByEntryDigestValue=4e5ee7a23e5b AssignmentRevisionsByTrunkRevisionValue=8553b390118d BusAnswer=7538f2f5e68e CatalogOf=7a4646fdbea7 Change=28d1f3c5cd32 ChangeAfterItem=30d718c21581 CompositionReader=5a943c1f8ef1 CompositionRun=a64bc503c99a CompositionRunProgram=5a1202df3497 CompositionsAnswer=f8bca93c7f9c CompositionsByGroupHeadsValue=9428965fab8f CompositionsDropAnswer=52803700282a Difference=4011cff516b1 DifferenceAnswer=629237aa39f0 DifferenceEvent=49066d0dff6d DifferenceRefusedItem=0e9caa83ace7 DifferencesByNodeModuleValue=a22487ffadfd DropAnswer=7538f2f5e68e Edge=a71709025a08 EntryAnswer=05adebad70d6 Fill=80f442916429 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 Held=15abbd6a1a33 HeldCandidate=04872f6afbfa InputAnswer=cd41ee65565d NodeEntryRef=3888ae847653 NotRunning=279b628db0ec PipelineWanted=e1e379c21e59 PipelineWantedEvent=59946b31ab7d PolicyChangedEvent=7538f2f5e68e RevisionAnswer=39002c770cbd RoomEvent=5bd4df78ed84 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 StepAccept=18efeefb6c34 StepEnded=edd08ab1a18b StepEndedEvent=207787118dac StepProgressEvent=8215e490ea8d StepRun=b975f402601a UpgradeAnswer=7538f2f5e68e Wait=951500be820c WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 WantedEvent=58cf7a9f646b WantsByHeldModuleFingerprintValue=ff14aa998a5d WantsByWantNodeModuleValue=ab7c6799fd5d WantsByWantNodeNodeValue=a737a2108d53" + generatedFieldsDigest = "Ask=6cb8b82e2fbe AskTakenEvent=6d5d3da7ac39 AsksByAssignmentsProposalValue=0bb5682c7b3b AsksByBackupNodeModuleFingerprintValue=0bb5682c7b3b AsksByHoldModuleFingerprintValue=0bb5682c7b3b AsksByOutsideNodeModuleFingerprintValue=0bb5682c7b3b AsksByRecordNodeModuleDigestValue=0bb5682c7b3b AsksByRestoreNodeModuleFingerprintValue=0bb5682c7b3b AsksByRevertNodeModuleFingerprintValue=0bb5682c7b3b AssignmentEntry=7d2549782eb2 AssignmentRevision=72c8701f7314 AssignmentRevisionCapabilitiesItem=0c89b0717762 AssignmentRevisionEvent=7faff968ce44 AssignmentRevisionsByCandidateGroupHeadsValue=8553b390118d AssignmentRevisionsByEntryDigestValue=4e5ee7a23e5b AssignmentRevisionsByTrunkRevisionValue=8553b390118d BusAnswer=7538f2f5e68e CatalogOf=7a4646fdbea7 Change=28d1f3c5cd32 ChangeAfterItem=30d718c21581 CompositionReader=5a943c1f8ef1 CompositionRun=a64bc503c99a CompositionRunProgram=5a1202df3497 CompositionsAnswer=f8bca93c7f9c CompositionsByGroupHeadsValue=9428965fab8f CompositionsDropAnswer=52803700282a Difference=4011cff516b1 DifferenceAnswer=629237aa39f0 DifferenceEvent=49066d0dff6d DifferenceRefusedItem=0e9caa83ace7 DifferencesByNodeModuleValue=a22487ffadfd DropAnswer=7538f2f5e68e Edge=a71709025a08 EntryAnswer=05adebad70d6 Fill=80f442916429 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 Held=15abbd6a1a33 HeldCandidate=04872f6afbfa InputAnswer=cd41ee65565d KeepNowAnswer=9962ab9fc051 NodeEntryRef=3888ae847653 NotRunning=279b628db0ec PipelineWanted=e1e379c21e59 PipelineWantedEvent=59946b31ab7d PolicyChangedEvent=7538f2f5e68e RevisionAnswer=39002c770cbd RoomEvent=5bd4df78ed84 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 StepAccept=18efeefb6c34 StepEnded=edd08ab1a18b StepEndedEvent=207787118dac StepProgressEvent=8215e490ea8d StepRun=b975f402601a UpgradeAnswer=7538f2f5e68e Wait=951500be820c WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 WantedEvent=58cf7a9f646b WantsByHeldModuleFingerprintValue=ff14aa998a5d WantsByWantNodeModuleValue=ab7c6799fd5d WantsByWantNodeNodeValue=a737a2108d53" ) const ( @@ -32,6 +32,7 @@ const ( VerbInput = "input" VerbCompositions = "compositions" VerbCompositionsDrop = "compositions-drop" + VerbKeepNow = "keep-now" VerbDrop = "drop" ) @@ -57,7 +58,7 @@ const ( // Verbs is every verb the seat serves, in its definition's order: a new copy each call. func Verbs() []string { - return []string{VerbUpgrade, VerbBus, VerbRevision, VerbEntry, VerbDifference, VerbInput, VerbCompositions, VerbCompositionsDrop, VerbDrop} + return []string{VerbUpgrade, VerbBus, VerbRevision, VerbEntry, VerbDifference, VerbInput, VerbCompositions, VerbCompositionsDrop, VerbKeepNow, VerbDrop} } // VerbDescription is a verb's description as the seat's definition says it, "" for a verb the seat does not serve or @@ -80,8 +81,10 @@ func VerbDescription(verb string) string { return "Each composition run the assigner keeps (design 47 G §728): its program, input digest, ask, readers, outcome and verdict, and the state's fill against its cap; with candidate or machine, only those." case VerbCompositionsDrop: return "Drop one composition run by hand (design 47 G §728): the act and why are written in the hand-act log first, then the run's ask is dropped, each reader ended could not run naming the drop, and the key deleted." + case VerbKeepNow: + return "Keep the assignment graph's revisions now, by hand (design 47 H2 §813 (2); H-A-417): written in the hand-act log first by the module's tool assigner_keep_now, then the keeping runs at once on the pass's newest computed revision; answers the revision kept, or the fault the keeping met." case VerbDrop: - return "End an ask on the seat's work queue for a caller that no longer waits for it (its step ended by its bound): one waiting is ended dropped when it is pulled, one taken is cancelled; both end the step dropped, naming who dropped it and why, so no holder works on it after (design 47 H2 §778; ADR 0664 §4)." + return "End an ask on the seat's work queue for a caller that no longer waits for it (its step ended by its bound): one waiting is ended dropped when it is pulled, one taken is cancelled; both end the step could-not-run, its why naming the drop and who asked it, so no holder works on it after (design 47 H2 §778; ADR 0664 §4)." } return "" } @@ -1272,6 +1275,22 @@ type CompositionsDropAnswer struct { Run CompositionRun `json:"run"` } +// KeepNowInput is what keep-now takes. +type KeepNowInput struct { + // why the keeping runs now + Why string `json:"why"` +} + +// KeepNowAnswer is what keep-now answers. +type KeepNowAnswer struct { + // the fault the keeping met; absent when it kept the revision + Fault string `json:"fault,omitempty"` + // is the form of what this value carries, left out while it is 1 (design 47 H2 §780). + Form int64 `json:"form,omitempty"` + // the revision kept + Revision *AssignmentRevision `json:"revision,omitempty"` +} + // DropInput is what drop takes. type DropInput struct { // who drops it @@ -1583,6 +1602,15 @@ func DecodeCompositionsDropAnswer(raw []byte) (CompositionsDropAnswer, int, erro return v, formRead(raw), nil } +// DecodeKeepNowAnswer reads a KeepNowAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780). +func DecodeKeepNowAnswer(raw []byte) (KeepNowAnswer, int, error) { + var v KeepNowAnswer + if err := json.Unmarshal(raw, &v); err != nil { + return v, 0, err + } + return v, formRead(raw), nil +} + // DecodeDropAnswer reads a DropAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780). func DecodeDropAnswer(raw []byte) (DropAnswer, int, error) { var v DropAnswer diff --git a/modules/mesh-delivery-contracts/seats/mesh-delivery/seat.json b/modules/mesh-delivery-contracts/seats/mesh-delivery/seat.json index eb8d844e3..08e74ab4e 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-delivery/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-delivery/seat.json @@ -649,6 +649,10 @@ "type": "string", "format": "date-time", "description": "when the gate first said it" + }, + "record": { + "type": "string", + "description": "the delivery record the act is for, by its source revision, .; absent on an act said before it was named, which bridge_owed_again by record never finds (design 47 H2 §807, H-A-432)" } }, "required": [ diff --git a/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json b/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json index e13f7058b..f9d926694 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json @@ -278,9 +278,39 @@ ] } }, + { + "name": "keep-now", + "description": "Keep the assignment graph's revisions now, by hand (design 47 H2 §813 (2); H-A-417): written in the hand-act log first by the module's tool assigner_keep_now, then the keeping runs at once on the pass's newest computed revision; answers the revision kept, or the fault the keeping met.", + "input": { + "type": "object", + "properties": { + "why": { + "type": "string", + "description": "why the keeping runs now" + } + }, + "required": [ + "why" + ] + }, + "answer": { + "type": "object", + "properties": { + "revision": { + "$ref": "#/defs/AssignmentRevision", + "description": "the revision kept" + }, + "fault": { + "type": "string", + "description": "the fault the keeping met; absent when it kept the revision" + } + } + }, + "class": "act" + }, { "name": "drop", - "description": "End an ask on the seat's work queue for a caller that no longer waits for it (its step ended by its bound): one waiting is ended dropped when it is pulled, one taken is cancelled; both end the step dropped, naming who dropped it and why, so no holder works on it after (design 47 H2 §778; ADR 0664 §4).", + "description": "End an ask on the seat's work queue for a caller that no longer waits for it (its step ended by its bound): one waiting is ended dropped when it is pulled, one taken is cancelled; both end the step could-not-run, its why naming the drop and who asked it, so no holder works on it after (design 47 H2 §778; ADR 0664 §4).", "input": { "type": "object", "properties": { diff --git a/modules/mesh-delivery-contracts/ts/seats/meshdelivery/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshdelivery/index.d.ts index ee0549577..f07a02274 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshdelivery/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshdelivery/index.d.ts @@ -110,6 +110,8 @@ export interface Owed { "line"?: string; /** the pull request's number */ "number": number; + /** the delivery record the act is for, by its source revision, .; absent on an act said before it was named, which bridge_owed_again by record never finds (design 47 H2 §807, H-A-432) */ + "record"?: string; /** the repository, / */ "repository": string; /** when the gate first said it */ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts index b9f4ef056..dd3efa85d 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts @@ -13,6 +13,7 @@ export declare const verbs: { readonly input: "input"; readonly compositions: "compositions"; readonly compositionsDrop: "compositions-drop"; + readonly keepNow: "keep-now"; readonly drop: "drop"; }; @@ -672,6 +673,22 @@ export interface CompositionsDropAnswer { "run": CompositionRun; } +/** is what keep-now takes. */ +export interface KeepNowInput { + /** why the keeping runs now */ + "why": string; +} + +/** is what keep-now answers. */ +export interface KeepNowAnswer { + /** the fault the keeping met; absent when it kept the revision */ + "fault"?: string; + /** is the form of what this value carries, left out while it is 1 (design 47 H2 §780). */ + "form"?: number; + /** the revision kept */ + "revision"?: AssignmentRevision; +} + /** is what drop takes. */ export interface DropInput { /** who drops it */ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js index 7c9016b74..94fb35ed8 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js @@ -13,6 +13,7 @@ export const verbs = { input: "input", compositions: "compositions", compositionsDrop: "compositions-drop", + keepNow: "keep-now", drop: "drop", }; diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go index 560d5659e..84d5bbdb6 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go @@ -16,6 +16,8 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/stateobjects" + "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-forge-bridge/internal/bridge" "mesh-delivery-forge-bridge/internal/groups" @@ -126,6 +128,27 @@ func askState(s stdio.KeptState, bucket string, bound time.Duration, done func([ } } +// Module is the module whose tools these are; the generic prune tools are named for it (H-A-380 (3), H-A-433). +const Module = "mesh-delivery-forge-bridge" + +// PerformedPruneTools are `mesh-delivery-forge-bridge_state_prune_now` and `mesh-delivery-forge-bridge_state_keep` over +// the state performed (design 47 H2 §812): a prune to prune-target now, and a key kept from the prune under _keep/ +// until a time, each written in the hand-act log first. +func PerformedPruneTools(performed stdio.Prunable, acts views.HandActs) []stdio.Tool { + return stateobjects.PruneTools(Module, []stateobjects.Pruning{{Name: groups.PerformedState, State: performed}}, acts) +} + +// prunedPerformed is the state performed as the prune tools act on it, with the module's prune-mark and prune-target; a +// setting that is no percentage is said, and the defaults hold. +func prunedPerformed() *stdio.PrunedState { + p, err := stdio.PruneSettings() + if err != nil { + fmt.Fprintf(os.Stderr, "[mesh-delivery-forge-bridge-tools] the prune of performed keeps its defaults (%d %%, to %d %%): %v\n", stdio.PruneMarkDefault, stdio.PruneTargetDefault, err) + p = stdio.Prune{Mark: stdio.PruneMarkDefault, Target: stdio.PruneTargetDefault} + } + return stdio.State(groups.PerformedState).Pruned(p) +} + // askCaller calls a tool as this bundle's module, through the runtime (stdio.Ask), the module's own invokes granting it. func askCaller(_ context.Context, key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) @@ -141,10 +164,11 @@ func main() { tools = append(tools, OwedTools(askState(owedAsks, groups.OwedAsksBucket(served), 25*time.Second, func(v []byte) bool { a, err := groups.DecodeOwedAsk(v) return err == nil && a.Done() - }), func(f OwedFilter) ([]OwedRow, error) { return performedRows(performed, f) }, func(what, why string) error { - _, err := recorder.Record(meshhandactsclient.Request{What: what, Why: why, Cause: "owed-dropped"}) + }), func(f OwedFilter) ([]OwedRow, error) { return performedRows(performed, f) }, func(what, why, cause string) error { + _, err := recorder.Record(meshhandactsclient.Request{What: what, Why: why, Cause: cause}) return err }, time.Now)...) + tools = append(tools, PerformedPruneTools(prunedPerformed(), recorder)...) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-forge-bridge-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go index 832f48ff5..8291cf46a 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go @@ -4,12 +4,14 @@ import ( "encoding/json" "errors" "reflect" + "strings" "testing" "time" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-forge-bridge/internal/groups" ) @@ -110,7 +112,10 @@ func TestTheBridgeTools(t *testing.T) { } var recorded []string recordErr := error(nil) - record := func(what, why string) error { recorded = append(recorded, what+": "+why); return recordErr } + record := func(what, why, cause string) error { + recorded = append(recorded, what+": "+why+" ("+cause+")") + return recordErr + } tools := map[string]func(map[string]any) (any, error){} for _, tl := range OwedTools(ask, performed, record, func() time.Time { return at }) { tools[tl.Name] = tl.Run @@ -163,5 +168,59 @@ func TestTheBridgeTools(t *testing.T) { if _, _, err := groups.NewOwedAsk(groups.OwedAgain, "", "a", "", "", at); err == nil { t.Error("an ask naming no seat was taken") } + // Design 47 H2 §807 (H-A-432 (1)): bridge_owed_again by record asks the bridge for every act of one delivery record, + // written in the hand-act log first; with no why, or with an act named beside it, nothing is asked. + n, r := len(asked), len(recorded) + for _, bad := range []map[string]any{{"record": "main.212"}, {"record": "main.212", "of": seat.Seat, "id": "a", "why": "x"}} { + if _, err := tools[OwedAgainTool](bad); err == nil || len(asked) != n || len(recorded) != r { + t.Errorf("bridge_owed_again %v: %v, asked %d; want refused, nothing asked", bad, err, len(asked)-n) + } + } + if out, err := tools[OwedAgainTool](map[string]any{"record": "main.212", "why": "the forge was away"}); err != nil || + len(asked) != n+1 || asked[n].Record != "main.212" || asked[n].Kind != groups.OwedAgain || asked[n].ID != "" || + len(recorded) != r+1 || !strings.HasSuffix(recorded[r], "(owed-again)") { + t.Errorf("bridge_owed_again by record: %v %v, asked %+v, recorded %v", out, err, asked[n:], recorded[r:]) + } + }) + // Design 47 H2 §812 (H-A-380 (3), H-A-433): the bridge serves the two prune tools over its state performed, named for + // its module, each written in the hand-act log before it acts. + t.Run("the prune tools over performed", func(t *testing.T) { + p := &prunable{} + var recorded []views.HandAct + tools := map[string]stdio.Tool{} + for _, tool := range PerformedPruneTools(p, handActs(func(a views.HandAct) error { recorded = append(recorded, a); return nil })) { + tools[tool.Name] = tool + } + now, keep := tools["mesh-delivery-forge-bridge_state_prune_now"], tools["mesh-delivery-forge-bridge_state_keep"] + if len(tools) != 2 || now.Run == nil || keep.Run == nil { + t.Fatalf("the prune tools are %v, want mesh-delivery-forge-bridge_state_prune_now and _state_keep", tools) + } + until := time.Now().Add(time.Hour).UTC().Format(time.RFC3339) + if _, err := now.Run(map[string]any{"entry": "performed", "why": "full"}); err != nil || p.pruned != 1 { + t.Errorf("prune now: %v, pruned %d times", err, p.pruned) + } + if _, err := keep.Run(map[string]any{"entry": "performed", "key": "mesh-delivery.YQ", "until": until, "why": "read it"}); err != nil || p.kept != "mesh-delivery.YQ" { + t.Errorf("keep: %v, kept %q", err, p.kept) + } + if len(recorded) != 2 { + t.Errorf("hand acts %+v, want the prune and the keep", recorded) + } }) } + +// prunable is a pruned state in memory. +type prunable struct { + pruned int + kept string +} + +func (p *prunable) PruneNow() (stdio.Pruned, error) { + p.pruned++ + return stdio.Pruned{State: "performed"}, nil +} +func (p *prunable) Keep(key string, _ time.Time) error { p.kept = key; return nil } + +// handActs records a hand act through record. +type handActs func(views.HandAct) error + +func (h handActs) RecordHandAct(a views.HandAct) error { return h(a) } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/owed.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/owed.go index 646861c44..73457f2a0 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/owed.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/owed.go @@ -72,9 +72,9 @@ func performedRows(s stdio.KeptState, f OwedFilter) ([]OwedRow, error) { } // OwedTools are bridge_owed, bridge_owed_again and bridge_owed_drop: ask writes an owed-asks value and waits for the -// bridge's answer; performed reads the state performed; record writes a hand act in the seat mesh-hand-acts' log before -// a drop is asked, so a drop is never made without its record. -func OwedTools(ask func(key string, value []byte) ([]byte, error), performed func(OwedFilter) ([]OwedRow, error), record func(what, why string) error, now func() time.Time) []stdio.Tool { +// bridge's answer; performed reads the state performed; record writes a hand act in the seat mesh-hand-acts' log (what, +// why and its cause) before a drop or a record's acts performed again is asked, so neither is made without its record. +func OwedTools(ask func(key string, value []byte) ([]byte, error), performed func(OwedFilter) ([]OwedRow, error), record func(what, why, cause string) error, now func() time.Time) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } act := map[string]any{ "of": map[string]any{"type": "string", "description": "the seat that said the act: mesh-delivery (the gate)"}, @@ -96,6 +96,12 @@ func OwedTools(ask func(key string, value []byte) ([]byte, error), performed fun for k, v := range act { read[k] = v } + againArgs := map[string]any{ + "record": map[string]any{"type": "string", "description": "instead of of and id: a delivery record's source revision, .; with why"}, + "why": map[string]any{"type": "string", "description": "with record: why its acts are performed again, in plain words"}} + for k, v := range act { + againArgs[k] = v + } withWhy := map[string]any{"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}} for k, v := range act { withWhy[k] = v @@ -104,7 +110,7 @@ func OwedTools(ask func(key string, value []byte) ([]byte, error), performed fun {Name: OwedTool, Description: "What the forge bridge performed of what the gate and the checker owe the forge (design 47): each act it said as " + "owed.ended, newest first, with its seat, id, pull request, outcome (performed, refused, could not be performed, " + - "undecodable, dropped), the forge's words and when, from its state performed (kept 7 days), of one act (id, of " + + "undecodable, dropped, not-needed), the forge's words and when, from its state performed (kept 3 days), of one act (id, of " + "narrowing it) or of one pull request (repository and number), never the whole state; and the backlog of its " + "durable on each seat's stream owed: the last event applied, the events waiting and held, the age of the oldest. " + "Read-only.", @@ -133,9 +139,33 @@ func OwedTools(ask func(key string, value []byte) ([]byte, error), performed fun Description: "Perform one act owed to the forge again now, by hand, whatever its last outcome (a refusal the forge no longer " + "makes, an act that could not be performed): the bridge calls the seat git's verb of the act under its id (safe to " + "repeat) and says its outcome as owed.ended. The act is read from the stream owed of the seat that said it, on the " + - "subject of the pull request the bridge's state performed names (kept 7 days there). Answers the outcome.", - Input: map[string]any{"type": "object", "required": []string{"of", "id"}, "properties": act}, + "subject of the pull request the bridge's state performed names (kept 3 days there). Or, with record (a delivery " + + "record's source revision, .) and why instead of of and id, every act of that record that ended " + + "could not be performed, written first in the hand-act log (design 47 H2 §807). Each act reads its pull request " + + "again before it acts: one closed unmerged ends not-needed. Answers the outcome, or each act's.", + Input: map[string]any{"type": "object", "properties": againArgs}, Run: func(args map[string]any) (any, error) { + if rec := str(args, "record"); rec != "" { + if str(args, "of") != "" || str(args, "id") != "" { + return nil, fmt.Errorf("bridge_owed_again names one act (of and id) or one record, not both") + } + key, value, err := groups.NewOwedRecordAsk(rec, str(args, "why"), stdio.CallerOf(args), now()) + if err != nil { + return nil, err + } + if err := record("performed again every act of the record "+rec+" that could not be performed", str(args, "why"), "owed-again"); err != nil { + return nil, err + } + v, err := ask(key, value) + if err != nil { + return nil, err + } + a, err := groups.DecodeOwedAsk(v) + if err != nil { + return nil, err + } + return map[string]any{"record": a.Record, "answer": a.Answer}, nil + } a, err := asked(groups.OwedAgain, args) if err != nil { return nil, err @@ -151,7 +181,7 @@ func OwedTools(ask func(key string, value []byte) ([]byte, error), performed fun if str(args, "why") == "" { return nil, fmt.Errorf("a drop says why") } - if err := record(fmt.Sprintf("dropped the act %s that %s owes the forge", str(args, "id"), str(args, "of")), str(args, "why")); err != nil { + if err := record(fmt.Sprintf("dropped the act %s that %s owes the forge", str(args, "id"), str(args, "of")), str(args, "why"), "owed-dropped"); err != nil { return nil, err } a, err := asked(groups.OwedDrop, args) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound.go index f92d1ca7d..63fc0b06c 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound.go @@ -10,6 +10,7 @@ import ( "github.com/nats-io/nats.go/jetstream" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -38,10 +39,15 @@ type boundBucket struct { binding *process.Binding local string js process.BucketOpener + // prune, when set, prunes the entry toward its cap at the bridge's own writes (design 47 H2 §812; H-A-380): the + // bucket is bound through mesh-sdk's process.PrunedState, its figures read once when it is opened (again for a + // recreated entry's new bucket) and after each prune, and every Put is counted toward prune-mark. + prune *stdio.Prune - mu sync.Mutex - name string - kv jetstream.KeyValue + mu sync.Mutex + name string + kv jetstream.KeyValue + pruned *process.PrunedState } func (s *boundBucket) open(ctx context.Context) (jetstream.KeyValue, error) { @@ -61,7 +67,13 @@ func (s *boundBucket) open(ctx context.Context) (jetstream.KeyValue, error) { if err != nil { return nil, fmt.Errorf("the bus entry %q, bucket %s: %w", s.local, names.Bucket, err) } - s.name, s.kv = names.Bucket, kv + var pruned *process.PrunedState + if s.prune != nil { + if pruned, err = process.StateOver(forgeBridgeModule(), s.local, kv).Pruned(*s.prune); err != nil { + return nil, fmt.Errorf("the bus entry %q, bucket %s, cannot be bound for its prune: %w", s.local, names.Bucket, err) + } + } + s.name, s.kv, s.pruned = names.Bucket, kv, pruned return kv, nil } @@ -73,11 +85,19 @@ func (s *boundBucket) Get(ctx context.Context, key string) (jetstream.KeyValueEn return kv.Get(ctx, key) } +// Put writes a key whole; for an entry bound for its prune the write is counted, and past prune-mark the entry is +// pruned to prune-target before Put returns (§812). func (s *boundBucket) Put(ctx context.Context, key string, value []byte) (uint64, error) { kv, err := s.open(ctx) if err != nil { return 0, err } + s.mu.Lock() + pruned := s.pruned + s.mu.Unlock() + if pruned != nil { + return pruned.Put(key, json.RawMessage(value)) + } return kv.Put(ctx, key, value) } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound_test.go index d521d33a9..480d8a757 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound_test.go @@ -48,7 +48,7 @@ func performedIsOpenedByItsBinding(t *testing.T) { o, pub, kept := newOwing(f) o.waits = binding.Waiting gate := gitfixture.Seat(t, seat.Seat, "events", "owed is said, a status", "event:owed")[0] - if _, err := o.heard(context.Background(), seat.Seat, gate); err == nil || len(f.calls) != 0 || len(pub.data) != 0 || len(kept) != 0 { + if _, err := o.heard(context.Background(), seat.Seat, gate, nil); err == nil || len(f.calls) != 0 || len(pub.data) != 0 || len(kept) != 0 { t.Errorf("unbound: %v, called %v, said %d; want the act left unperformed and unsaid", err, f.calls, len(pub.data)) } answered = true diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go index 517a56d25..b324f7b95 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "log" + "strconv" "strings" "testing" "time" @@ -14,6 +15,8 @@ import ( seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + "git.novox.be/novox/mesh-sdk/go/busretry" + "mesh-delivery-forge-bridge/internal/gitfixture" "mesh-delivery-forge-bridge/internal/groups" ) @@ -38,6 +41,13 @@ func (m memPerformed) Get(_ context.Context, key string) ([]byte, bool, error) { return v, ok, nil } func (m memPerformed) Put(_ context.Context, key string, v []byte) error { m[key] = v; return nil } +func (m memPerformed) Keys(context.Context) ([]string, error) { + var out []string + for k := range m { + out = append(out, k) + } + return out, nil +} // memActs is a seat's stream owed by pull request subject (`#`); asked records what was read. type memActs struct { @@ -58,13 +68,28 @@ func (m *memActs) act(_ context.Context, of, repository string, number int, id s } // seatFake answers each seat verb from answers, by its key; one absent is not answered (an error, as a bound passed). +// The seat git's pull, read before each try of an act (H-A-432), is counted apart in pulls and answers an open pull +// request unless answers names it; away answers the act verbs `away` that many times first (§807). type seatFake struct { calls []string + pulls int + away int answers map[string]json.RawMessage } func (f *seatFake) call(_ context.Context, key string, _ any) (json.RawMessage, error) { + if key == "seat:git.pull" { + f.pulls++ + if a, ok := f.answers[key]; ok { + return a, nil + } + return json.RawMessage(`{"pull":{"repository":"novox/mesh-sdk","number":178,"branch":"b","head":"` + headA + `","base":"main","state":"open"}}`), nil + } f.calls = append(f.calls, key) + if f.away > 0 { + f.away-- + return json.Marshal(busretry.AwayAnswer("the forge", "401 while the store is replaced")) + } if a, ok := f.answers[key]; ok { return a, nil } @@ -104,11 +129,16 @@ func TestWhatIsOwedIsPerformedSaidAndKept(t *testing.T) { {"a seat that says no owed act", "mesh-wants", gate("owed is said, a status"), nil, nil, "", forgemsg.OwedUndecodable, false, ""}, {"the stream forge refusing the outcome", seat.Seat, gate("owed is said, a status"), statusOK, errors.New("no responders"), "seat:git.status", "", false, "could not be said"}, + // Design 47 H2 §807 (H-A-432 (2)): a pull request closed unmerged, read before the try, ends the act not-needed. + {"a status whose pull request closed unmerged", seat.Seat, gate("owed is said, a status"), + map[string]json.RawMessage{"seat:git.status": answer("status", "status answers"), + "seat:git.pull": json.RawMessage(`{"pull":{"repository":"novox/mesh-sdk","number":178,"branch":"b","head":"` + headA + `","base":"main","state":"closed"}}`)}, + nil, "", forgemsg.OwedNotNeeded, true, ""}, } { f := &seatFake{answers: c.answers} o, pub, kept := newOwing(f) pub.err = c.pubErr - _, err := o.heard(context.Background(), c.of, c.data) + _, err := o.heard(context.Background(), c.of, c.data, nil) if c.err != "" { if err == nil || !strings.Contains(err.Error(), c.err) || len(kept) != 0 { t.Errorf("%s: %v, kept %d; want an error naming %q and nothing kept", c.name, err, len(kept), c.err) @@ -138,6 +168,55 @@ func TestWhatIsOwedIsPerformedSaidAndKept(t *testing.T) { } } + // Design 47 H2 §807, §775: an act met by the seat git's away is called again at the kept-call pace, its pull request + // read before each try, and performed when the forge answers; one away past owed-away-bound ends could not be + // performed, naming every try. + t.Run("an act through a part away", func(t *testing.T) { + defer busretry.Waits(time.Minute, time.Millisecond, 2*time.Millisecond)() + f := &seatFake{answers: statusOK, away: 2} + o, pub, _ := newOwing(f) + if e, err := o.heard(context.Background(), seat.Seat, gate("owed is said, a status"), nil); err != nil || e.Outcome != forgemsg.OwedPerformed || + len(f.calls) != 3 || f.pulls != 3 || len(pub.data) != 1 { + t.Errorf("away twice: %+v %v, called %v, pulls %d; want performed at the third try, its pull request read before each", e, err, f.calls, f.pulls) + } + f = &seatFake{answers: statusOK, away: 1 << 30} + o, _, _ = newOwing(f) + o.awayBound = 10 * time.Millisecond + if e, err := o.heard(context.Background(), seat.Seat, gate("owed is said, a status"), nil); err != nil || e.Outcome != forgemsg.OwedCouldNotBePerformed || + !strings.Contains(e.Words, "called "+strconv.Itoa(len(f.calls))+" times") || len(f.calls) < 2 { + t.Errorf("away past the bound: %+v %v after %d calls; want could not be performed naming every try", e, err, len(f.calls)) + } + }) + + // Design 47 H2 §807 (H-A-432 (1)): bridge_owed_again by record performs again every act of that record that ended + // could not be performed, found in performed by the record kept with it; an act of it that ended otherwise, and one + // of another record, are left. + t.Run("a record's acts performed again", func(t *testing.T) { + f := &seatFake{answers: statusOK} + o, pub, kept := newOwing(f) + status := "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/merge-gate" + o.acts = &memActs{held: map[string][][]byte{"mesh-delivery novox/mesh-sdk#178": {gate("owed is said, a status")}}} + at := time.Date(2026, 11, 9, 20, 0, 0, 0, time.UTC) + for id, c := range map[string]struct{ record, outcome string }{ + status: {"main.212", forgemsg.OwedCouldNotBePerformed}, "done": {"main.212", forgemsg.OwedPerformed}, + "another": {"main.213", forgemsg.OwedCouldNotBePerformed}} { + kept[performedKey(seat.Seat, id)], _ = json.Marshal(Performed{Ended: forgemsg.OwedEnded{V: 1, Of: seat.Seat, ID: id, + Outcome: c.outcome, Words: "x", At: at}, Repository: "novox/mesh-sdk", Number: 178, Record: c.record}) + } + a := o.answer(context.Background(), groups.OwedAsk{Kind: groups.OwedAgain, Record: "main.212", Why: "the forge was away"}, nil) + if !a.Done() || !strings.Contains(a.Answer, "1 acts of the record main.212") || !strings.Contains(a.Answer, status+": performed") || + len(f.calls) != 1 || f.pulls != 1 || len(pub.data) != 1 { + t.Errorf("again by record: %+v, called %v, pulls %d, said %d; want the one act that could not be performed, performed", a, f.calls, f.pulls, len(pub.data)) + } + var p Performed + if err := json.Unmarshal(kept[performedKey(seat.Seat, status)], &p); err != nil || p.Ended.Outcome != forgemsg.OwedPerformed || p.Record != "main.212" { + t.Errorf("performed after again by record: %+v %v", p, err) + } + if a := o.answer(context.Background(), groups.OwedAsk{Kind: groups.OwedAgain, Record: "main.212"}, nil); !strings.Contains(a.Answer, "holds no act") { + t.Errorf("again by record, nothing left: %+v", a) + } + }) + t.Run("heard from a seat's stream owed", func(t *testing.T) { f := &seatFake{answers: statusOK} o, pub, _ := newOwing(f) @@ -168,7 +247,7 @@ func TestWhatIsOwedIsPerformedSaidAndKept(t *testing.T) { acts := &memActs{held: map[string][][]byte{"mesh-delivery novox/mesh-sdk#178": {[]byte(`{`), gate("owed is said, a status")}}} o.acts = acts status := "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/merge-gate" - if _, err := o.heard(context.Background(), seat.Seat, gate("owed is said, a status")); err != nil { + if _, err := o.heard(context.Background(), seat.Seat, gate("owed is said, a status"), nil); err != nil { t.Fatal(err) } backlog := func(context.Context) []groups.Backlog { return []groups.Backlog{{Stream: seat.StreamOwed, Waiting: 2}} } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go index 260c87765..fb1e081fa 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go @@ -22,6 +22,8 @@ import ( git "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" @@ -111,6 +113,8 @@ accounts { "mesh.seat.git.tool.note" "mesh.seat.git.call.tags.n1~mesh-delivery-forge-bridge" "mesh.seat.git.tool.tags" + "mesh.seat.git.call.pull.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.pull" "mesh.seat.mesh-hand-acts.call.hand-act.n1~mesh-delivery-forge-bridge" "mesh.seat.mesh-hand-acts.tool.hand-act" "mesh.stream.mesh-delivery-forge-bridge.event.forge.>" @@ -178,9 +182,17 @@ type bed struct { // answerOf is a verb's answer in the reply envelope of mesh-sdk go/process/call.go ({result, error, node}). func answerOf(answer string) []byte { return []byte(`{"result":` + answer + `,"node":"n0"}`) } +// pullAnswer is the seat git's pull answering one pull request in state (open, merged or closed). +func pullAnswer(repository string, number int, state string) string { + return fmt.Sprintf(`{"pull":{"repository":%q,"number":%d,"branch":"b","head":"%s","base":"main","state":%q}}`, repository, number, strings.Repeat("a", 40), state) +} + // boundPerformed is the bus name the stand-in seat mesh-broker's show answers for the state performed. const boundPerformed = "mod_mesh-delivery-forge-bridge_performed_b1" +// performedCap is the cap of the bed's performed, small so a test fills it to its prune-mark (design 47 H2 §812). +const performedCap = 64 << 10 + // catalog is the repository the stand-in registrations name, its branch main watched (the seat mesh-catalog's fixture). const catalog = "novox/mesh-catalog" @@ -216,7 +228,11 @@ func startBed(t *testing.T, head string) *bed { } } for _, name := range []string{headsBucket(forgeBridgeModule()), boundPerformed, groupkeys.OwedAsksBucket(forgeBridgeModule())} { - if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 1}); err != nil { + var capped int64 + if name == boundPerformed { + capped = performedCap + } + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 1, MaxBytes: capped}); err != nil { t.Fatal(err) } } @@ -241,6 +257,7 @@ func startBed(t *testing.T, head string) *bed { "mesh-delivery.source": `{"branches":["main"]}`, "mesh-broker.show": `{"state":"created","binding":{"local":"performed","storage":"key-value","template":"state","bucket":"` + boundPerformed + `"}}`, "git.branch " + catalog: `{"repository":"` + catalog + `","branch":"main","head":"` + head + `","protected":true}`, + "git.pull": pullAnswer("novox/mesh-sdk", 178, "open"), } for _, verb := range []string{"status", "comment", "view", "note"} { answers["git."+verb] = string(gitfixture.Given(t, verb+" answers", "verb:"+verb+".answer")) @@ -481,6 +498,10 @@ func refusedRead(t *testing.T, b *bed, bucket string) { // Acceptance 49: the forge bridge sets the group on each member's merged message and says the group's end once, after the // last member's, from the seat mesh-delivery's verb groups; its account reads no state of the gate's. func TestBridgePerformsWhatIsOwedOnce(t *testing.T) { + // The bridge's kept-call pace, read at its start (§775), in milliseconds here so the forge's minute away is short. + t.Cleanup(busretry.Waits(busretry.Bound, busretry.FirstWait, busretry.MostWait)) + t.Setenv(stdio.SettingWord(busretry.PaceFirstSetting), "10ms") + t.Setenv(stdio.SettingWord(busretry.PaceMaxSetting), "40ms") b := startBed(t, strings.Repeat("1", 40)) ch := heard(t, b) ended := func() forgemsg.OwedEnded { @@ -624,4 +645,143 @@ func TestBridgePerformsWhatIsOwedOnce(t *testing.T) { t.Errorf("performed of %s keeps %s, want the outcome, when and the pull request alone", g.ID, e.Value()) } refusedRead(t, b, "mesh-delivery-gate_owed") + + // Design 47 H2 §807 (F1, the bridge's whole test): with the forge away (the seat git answering away to every call), a + // note line owed in that time is performed once the forge answers again, with no hand act; an act whose pull request + // closed unmerged meanwhile ends not-needed, never performed (H-A-432 (2), (3)). + away, _ := json.Marshal(busretry.AwayAnswer("the forge", "401 while the store is replaced")) + for _, verb := range []string{"git.pull", "git.note", "git.comment"} { + b.answer(verb, string(away)) + } + handActs, pulls, comments := callsOf(b, "mesh-hand-acts.hand-act"), callsOf(b, "git.pull"), callsOf(b, "git.comment") + note := map[string]any{"id": "note.novox/mesh-sdk#178.aaaaaaaaaaaa.away", "act": "note", "repository": "novox/mesh-sdk", + "number": 178, "head": strings.Repeat("a", 40), "line": "built while the forge was away"} + comment := map[string]any{"id": "comment.novox/mesh-catalog#9.aaaaaaaaaaaa.away", "act": "comment", "repository": catalog, + "number": 9, "head": strings.Repeat("a", 40), "text": "closed meanwhile"} + for _, act := range []map[string]any{note, comment} { + data, _ := json.Marshal(act) + if _, err := b.js.Publish(b.ctx, seat.SubjectOwed(keys.EscapeName(act["repository"].(string)), fmt.Sprint(act["number"])), data); err != nil { + t.Fatal(err) + } + } + calledTimes(t, b, "git.pull", pulls+3) // tried, met away, and tried again at the pace + b.answer("git.pull "+"novox/mesh-sdk", pullAnswer("novox/mesh-sdk", 178, "merged")) + b.answer("git.pull "+catalog, pullAnswer(catalog, 9, "closed")) + b.answer("git.note", string(gitfixture.Given(t, "note answers", "verb:note.answer"))) + // Read until both acts ended (bridge_owed_again's outcome above is on the stream before them). + outcomes := map[string]forgemsg.OwedEnded{} + for i := 0; i < 4 && (outcomes[note["id"].(string)].ID == "" || outcomes[comment["id"].(string)].ID == ""); i++ { + e := ended() + outcomes[e.ID] = e + } + if e := outcomes[note["id"].(string)]; e.Outcome != forgemsg.OwedPerformed { + t.Errorf("the note owed while the forge was away: %+v, want performed once it answers", e) + } + if e := outcomes[comment["id"].(string)]; e.Outcome != forgemsg.OwedNotNeeded || !strings.Contains(e.Words, "closed unmerged") { + t.Errorf("the comment on a pull request closed meanwhile: %+v, want not-needed naming it", e) + } + if n := callsOf(b, "git.comment"); n != comments { + t.Errorf("the comment on a pull request closed meanwhile was asked of the forge %d times, want never", n-comments) + } + if n := callsOf(b, "mesh-hand-acts.hand-act"); n != handActs { + t.Errorf("%d hand acts while the forge was away, want none", n-handActs) + } + + // Design 47 H2 §812 (F2, the bridge's whole test): a write that takes performed past prune-mark (70 % of its cap) + // prunes it, oldest acts first, down to prune-target (50 %); a key kept under _keep/ survives the prune. performed is + // filled while the bridge is stopped, so the bridge's bind at its start reads the fill once (H-A-380 (1)). + fillKey := func(i int) string { return performedKey(seat.Seat, fmt.Sprintf("fill-%03d", i)) } + fillValue := func(i, pad int) []byte { + v, _ := json.Marshal(Performed{Ended: forgemsg.OwedEnded{V: forgemsg.Version, Of: seat.Seat, ID: fmt.Sprintf("fill-%03d", i), + Outcome: forgemsg.OwedRefused, Words: strings.Repeat("w", pad), At: time.Now().UTC()}, Repository: "novox/mesh-sdk", Number: 1}) + return v + } + fillOf := func() uint64 { + t.Helper() + st, err := kept.Status(b.ctx) + if err != nil { + t.Fatal(err) + } + return st.Bytes() + } + mark, target := uint64(performedCap*70/100), uint64(performedCap*50/100) + fills, lastPut := 0, uint64(0) + b.restart(func() { + // The oldest filler (written below) kept from the prune for an hour, as mesh-delivery-forge-bridge_state_keep writes it. + keep, _ := json.Marshal(map[string]any{"until": time.Now().Add(time.Hour).UTC()}) + if _, err := kept.Put(b.ctx, "_keep/"+fillKey(0), keep); err != nil { + t.Fatal(err) + } + // Fill to 150 bytes below the mark: the last filler's words make up what is left, by the bytes one write was seen to put. + for { + before := fillOf() + if before+1200 >= mark { + per := uint64(600) // a put's bytes beside its value, as the first filler showed + if fills > 0 { + per = lastPut - uint64(len(fillValue(fills-1, 300))) + } + pad := int(mark - 150 - before - per - uint64(len(fillValue(fills, 0)))) + if _, err := kept.Put(b.ctx, fillKey(fills), fillValue(fills, pad)); err != nil { + t.Fatal(err) + } + fills++ + break + } + if _, err := kept.Put(b.ctx, fillKey(fills), fillValue(fills, 300)); err != nil { + t.Fatal(err) + } + lastPut = fillOf() - before + fills++ + } + }) + if f := fillOf(); f >= mark || f+400 < mark { + t.Fatalf("performed is filled to %d bytes, want just below its mark %d", f, mark) + } + pruned, err := kept.Watch(b.ctx, fillKey(1), jetstream.UpdatesOnly()) + if err != nil { + t.Fatal(err) + } + defer pruned.Stop() + other["id"] = "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/prune" + pruning, _ := json.Marshal(other) + if _, err := b.js.Publish(b.ctx, subject, pruning); err != nil { + t.Fatal(err) + } + if e := ended(); e.Outcome != forgemsg.OwedPerformed { + t.Errorf("the act that takes performed past its mark: %+v, want performed", e) + } + for gone, bound := false, time.After(20*time.Second); !gone; { + select { + case e := <-pruned.Updates(): + gone = e != nil && e.Operation() == jetstream.KeyValuePurge + case <-bound: + t.Fatalf("performed past its mark (%d of %d bytes) was not pruned within 20 s", fillOf(), uint64(performedCap)) + } + } + held := func(key string) bool { + _, err := kept.Get(b.ctx, key) + return err == nil + } + if !held(fillKey(0)) || !held("_keep/"+fillKey(0)) { + t.Errorf("the act kept under _keep/ was pruned") + } + if !held(performedKey(seat.Seat, "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/prune")) || !held(fillKey(fills-1)) { + t.Errorf("the newest acts were pruned") + } + // What the current values hold is down to the target: the bytes left, counted by mesh-sdk's upper bound per write (its + // key, its value and PutOverhead), which reads a little above the bus's own bytes the prune counted down from, so within + // 4 % of the cap; each purged key's marker aside. The bucket as the bus counts it is below the mark again. + keysLeft, err := kept.ListKeys(b.ctx) + if err != nil { + t.Fatal(err) + } + var left uint64 + for k := range keysLeft.Keys() { + if e, err := kept.Get(b.ctx, k); err == nil { + left += uint64(len(k) + len(e.Value()) + 96) + } + } + if left > target+performedCap*4/100 || left < target/2 || fillOf() >= mark { + t.Errorf("performed holds %d bytes of current values after its prune, want about its target %d (cap %d)", left, target, uint64(performedCap)) + } } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index 5c3b26e6f..652911956 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -16,6 +16,8 @@ import ( "syscall" "time" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/views" @@ -57,10 +59,18 @@ type delivered interface { Data() []byte Headers() nats.Header Ack() error + InProgress() error Term() error NakWithDelay(delay time.Duration) error } +// owedAwayBoundSetting is the setting owed-away-bound: how long an owed act is called again through a part away before +// it ends could not be performed (design 47 H2 §807), 60 minutes when none is composed. +const ( + owedAwayBoundSetting = "owed-away-bound" + owedAwayBoundDefault = 60 * time.Minute +) + // retryDelay is how long the server waits before it delivers again an event the forge could not be asked about (the // server's redelivery is the only clock; issue 1236/AC3). const retryDelay = 10 * time.Second @@ -106,9 +116,28 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if w := binding.Waiting(); w != "" { logger.Printf("the bridge %s: nothing owed to the forge is performed until it is bound", w) } - performedKV := &boundBucket{binding: binding, local: performedLocal, js: js} + // Each owed act is a kept call (design 47 H2 §807, §775): at the module's kept-call pace, until owed-away-bound. A + // setting that is no duration is said, and its default holds. + if err := busretry.ReadPace(stdio.SettingDuration); err != nil { + logger.Printf("the kept-call pace keeps %s doubling to %s: %v", busretry.KeptPace.First, busretry.KeptPace.Max, err) + } + awayBound, err := stdio.SettingDuration(owedAwayBoundSetting, owedAwayBoundDefault) + if err != nil { + logger.Printf("owed-away-bound keeps its default %s: %v", owedAwayBoundDefault, err) + awayBound = owedAwayBoundDefault + } + // performed is pruned toward its cap at the bridge's own writes (design 47 H2 §812): past prune-mark the oldest acts + // go down to prune-target, in one journal line, a key kept under _keep/ staying. A setting that is no percentage is + // said, and the defaults hold. + prune, err := stdio.PruneSettings() + if err != nil { + logger.Printf("the prune of performed keeps its defaults (%d %%, to %d %%): %v", stdio.PruneMarkDefault, stdio.PruneTargetDefault, err) + prune = stdio.Prune{Mark: stdio.PruneMarkDefault, Target: stdio.PruneTargetDefault} + } + performedKV := &boundBucket{binding: binding, local: performedLocal, js: js, prune: &prune} // What the gate owes the forge, performed through the seat git (design 47 *What the gate owes the forge*). - owe := &owing{call: call, pub: js, performed: kvPerformed{performedKV}, acts: streamActs{js: js}, waits: binding.Waiting, now: time.Now, logger: logger} + owe := &owing{call: call, pub: js, performed: kvPerformed{performedKV}, acts: streamActs{js: js}, waits: binding.Waiting, + awayBound: awayBound, now: time.Now, logger: logger} keptCall := keptCallOf(ctx, call) reader := &views.ProcessReader{Module: forgeBridgeModule(), Call: keptCall, Logf: logger.Printf, Hub: hub} src := sourcebranches.New(sourcebranches.Call(keptCall)) @@ -182,7 +211,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // What the gate says it owes, on its seat's stream owed through this module's durable there (made from its consumes). // A stream not there yet is said, and the bridge goes on: its other work does not wait on the gate. if cc, err := cred.HearSeatEvents(ctx, js, seat.StreamOwed, func(m jetstream.Msg) { - hctx, cancel := context.WithTimeout(ctx, handleDeadline) + hctx, cancel := context.WithTimeout(ctx, awayBound+handleDeadline) // an act waits out a part away until owed-away-bound defer cancel() owe.hear(hctx, seat.Seat, m) }, logger.Printf); err != nil { @@ -196,7 +225,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro backlog := owedBacklog(js, cred, time.Now) go func() { err := watch(ctx, kv, func(key string, rev uint64, value []byte) { - hctx, cancel := context.WithTimeout(ctx, handleDeadline) + hctx, cancel := context.WithTimeout(ctx, awayBound+handleDeadline) // each act performed again is a kept call defer cancel() owe.asked(hctx, kv, key, rev, value, backlog) }) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go index 80475ec05..088996b86 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go @@ -30,6 +30,7 @@ func (f *fakeMsg) Subject() string { return f.subject } func (f *fakeMsg) Data() []byte { return f.data } func (f *fakeMsg) Headers() nats.Header { return nats.Header{} } func (f *fakeMsg) Ack() error { f.acked++; return nil } +func (f *fakeMsg) InProgress() error { return nil } func (f *fakeMsg) NakWithDelay(d time.Duration) error { f.nak = append(f.nak, d); return nil } func (f *fakeMsg) Term() error { f.termd++; return nil } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go index 8999e70a2..8a999ed9f 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go @@ -6,6 +6,8 @@ import ( "errors" "fmt" "log" + "sort" + "strings" "time" "github.com/nats-io/nats.go/jetstream" @@ -14,6 +16,7 @@ import ( seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-forge-bridge/internal/bridge" @@ -30,13 +33,16 @@ func performedKey(of, id string) string { return groups.PerformedKey(of, id) } // Performed is one value of the state performed. type Performed = groups.Performed -// performedStore is the state performed as the owing reads and writes it. +// performedStore is the state performed as the owing reads and writes it; Keys lists it for bridge_owed_again by record. type performedStore interface { Get(ctx context.Context, key string) (value []byte, found bool, err error) Put(ctx context.Context, key string, value []byte) error + Keys(ctx context.Context) ([]string, error) } -type kvPerformed struct{ kv keptStore } +type kvPerformed struct{ kv *boundBucket } + +func (s kvPerformed) Keys(ctx context.Context) ([]string, error) { return s.kv.Keys(ctx) } func (s kvPerformed) Get(ctx context.Context, key string) ([]byte, bool, error) { e, err := s.kv.Get(ctx, key) @@ -64,9 +70,12 @@ type owing struct { // acts reads an act again from the stream owed of the seat that said it (H-A-181), for bridge_owed_again. acts actReader // waits, when set, says why the state performed cannot be used yet (its binding, design 47 G §577); empty when it can. - waits func() string - now func() time.Time - logger *log.Logger + waits func() string + // awayBound is owed-away-bound: how long an act is called again through transport faults and the seat git's `away` + // before it ends could not be performed, naming its tries (design 47 H2 §807, §775); zero is busretry.Bound. + awayBound time.Duration + now func() time.Time + logger *log.Logger } // readerOf is the reading of the seat of's owed event into the seat git's verb. @@ -80,15 +89,17 @@ func readerOf(of string) (func([]byte) (bridge.OwedCall, error), error) { // heard performs one owed event the seat of said. Its error is only an outcome that could not be said on the stream // forge: the event is then heard again, and the act, safe to repeat, performed again. -func (o *owing) heard(ctx context.Context, of string, data []byte) (forgemsg.OwedEnded, error) { +// tried, when set, is told before each try of the act's kept call (the message's progress, so the bus does not hand it +// again while the act waits out a part away). +func (o *owing) heard(ctx context.Context, of string, data []byte, tried func()) (forgemsg.OwedEnded, error) { read, err := readerOf(of) if err != nil { - return o.end(ctx, forgemsg.OwedEnded{Of: of, Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, "", 0) + return o.end(ctx, forgemsg.OwedEnded{Of: of, Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, bridge.OwedCall{}) } c, err := read(data) if err != nil { return o.end(ctx, forgemsg.OwedEnded{Of: of, ID: c.ID, - Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, c.Repository, c.Number) + Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, c) } if w := o.waiting(); w != "" { return forgemsg.OwedEnded{Of: c.Of, ID: c.ID}, fmt.Errorf("the bridge %s", w) @@ -97,13 +108,13 @@ func (o *owing) heard(ctx context.Context, of string, data []byte) (forgemsg.Owe // said twice, or said again while its outcome was on its way: performed once, the outcome said once return forgemsg.OwedEnded{Of: c.Of, ID: c.ID}, nil } - return o.perform(ctx, c) + return o.perform(ctx, c, tried) } // hear is one message of a seat's stream owed: performed and acknowledged, or, when its outcome could not be said, // left to come again (design 47: the bridge stopped between an act and its owed.ended performs it again). func (o *owing) hear(ctx context.Context, of string, m delivered) { - if _, err := o.heard(ctx, of, m.Data()); err != nil { + if _, err := o.heard(ctx, of, m.Data(), func() { _ = m.InProgress() }); err != nil { o.logger.Printf("%s, so the act is heard again: %v", m.Subject(), err) _ = m.NakWithDelay(retryDelay) return @@ -119,21 +130,77 @@ func (o *owing) waiting() string { return o.waits() } -// perform calls the seat git's verb of one act and says what it came to. -func (o *owing) perform(ctx context.Context, c bridge.OwedCall) (forgemsg.OwedEnded, error) { - answer, callErr := o.call(ctx, "seat:"+git.Seat+"."+c.Verb, c.Input) - outcome, words := bridge.Outcome(answer, callErr) +// perform calls the seat git's verb of one act as a kept call (design 47 H2 §807, §775) and says what it came to: through +// transport faults and the seat's `away` it is called again at the kept-call pace until owed-away-bound, then ends could +// not be performed naming every try. Before each try its precondition is read again (H-A-432 (2), (3)): an act whose +// pull request closed unmerged ends not-needed, never performed. +func (o *owing) perform(ctx context.Context, c bridge.OwedCall, tried func()) (forgemsg.OwedEnded, error) { + type try struct { + answer json.RawMessage + closed string + } + t, callErr := busretry.DoPaced(busretry.Kept{Seat: git.Seat, Verb: c.Verb, Ask: c.ID, Bound: o.awayBound, Logf: o.logger.Printf}, func() (try, error) { + if tried != nil { + tried() + } + closed, err := o.closed(ctx, c) + if err != nil || closed != "" { + return try{closed: closed}, err + } + answer, err := o.call(ctx, "seat:"+git.Seat+"."+c.Verb, c.Input) + if err == nil { + if away := busretry.AwayIn(answer); away != nil { + return try{answer: answer}, away + } + } + return try{answer: answer}, err + }) + outcome, words := bridge.Outcome(t.answer, callErr) + if callErr == nil && t.closed != "" { + outcome, words = forgemsg.OwedNotNeeded, t.closed + } if outcome != forgemsg.OwedPerformed { o.logger.Printf("the owed %s %s of %s#%d (said by %s) is %s: %s", c.Act, c.ID, c.Repository, c.Number, c.Of, outcome, words) } return o.end(ctx, forgemsg.OwedEnded{Of: c.Of, ID: c.ID, - Outcome: outcome, Words: words}, c.Repository, c.Number) + Outcome: outcome, Words: words}, c) +} + +// closed is why the act is not needed: its pull request closed unmerged, as the seat git's pull answers it now (open +// and merged both need it, since the gate owes notes and views on merged pull requests; H-A-432 (2)); empty when it is +// needed. An act that names no pull request (a status said on `.0`) has none to read. A pull the forge does not answer +// is the try's transport fault, called again; one it refuses is the act's error, never performed unread. +func (o *owing) closed(ctx context.Context, c bridge.OwedCall) (string, error) { + if c.Number <= 0 { + return "", nil + } + raw, err := o.call(ctx, "seat:"+git.Seat+"."+git.VerbPull, git.PullInput{Repository: c.Repository, Number: int64(c.Number)}) + if err == nil { + if away := busretry.AwayIn(raw); away != nil { + err = away + } + } + if err != nil { + return "", fmt.Errorf("the pull request %s#%d could not be read before the act: %w", c.Repository, c.Number, err) + } + var a git.PullAnswer + switch { + case json.Unmarshal(raw, &a) != nil: + return "", fmt.Errorf("the seat git's pull of %s#%d answered what its contract cannot read: %s", c.Repository, c.Number, raw) + case a.Error != "": + return "", fmt.Errorf("the pull request %s#%d could not be read before the act: %s", c.Repository, c.Number, a.Error) + case a.Pull == nil: + return "", fmt.Errorf("the seat git's pull of %s#%d answered no pull request", c.Repository, c.Number) + case a.Pull.State == git.PullStateClosed: + return fmt.Sprintf("the pull request %s#%d closed unmerged, so the %s is not needed", c.Repository, c.Number, c.Act), nil + } + return "", nil } // end says an act's outcome as owed.ended on the stream forge, then keeps it in performed with the pull request the act -// names (repository and number). An outcome not kept is logged: the act may be performed again, which changes nothing at -// the forge. -func (o *owing) end(ctx context.Context, e forgemsg.OwedEnded, repository string, number int) (forgemsg.OwedEnded, error) { +// names (repository and number) and its delivery record. An outcome not kept is logged: the act may be performed again, +// which changes nothing at the forge. +func (o *owing) end(ctx context.Context, e forgemsg.OwedEnded, c bridge.OwedCall) (forgemsg.OwedEnded, error) { e.At = o.now().UTC() data, err := forgemsg.EncodeOwedEnded(e) if err != nil { @@ -145,7 +212,7 @@ func (o *owing) end(ctx context.Context, e forgemsg.OwedEnded, repository string if e.ID == "" { return e, nil // an undecodable act with no id is said, and has no key to keep } - v, err := json.Marshal(Performed{Ended: e, Repository: repository, Number: number}) + v, err := json.Marshal(Performed{Ended: e, Repository: c.Repository, Number: c.Number, Record: c.Record}) if err == nil { err = o.performed.Put(ctx, performedKey(e.Of, e.ID), v) } @@ -179,14 +246,19 @@ func (o *owing) kept(ctx context.Context, of, id string) (Performed, error) { return p, nil } -// answer answers one owed ask: an act performed again now, whatever performed holds of it; an act dropped, said as -// owed.ended with the outcome dropped and the ask's why (the tool wrote the hand-act log first); or the consumers' backlog. +// answer answers one owed ask: an act performed again now, whatever performed holds of it; every act of one delivery +// record that ended could not be performed, performed again (design 47 H2 §807, H-A-432 (1)); an act dropped, said as +// owed.ended with the outcome dropped and the ask's why (the tool wrote the hand-act log first); or the consumers' +// backlog. Each act performed again reads its precondition again before it acts (perform). func (o *owing) answer(ctx context.Context, a groups.OwedAsk, backlog func(context.Context) []groups.Backlog) groups.OwedAsk { a.Answered = o.now().UTC() - switch a.Kind { - case groups.OwedBacklog: + switch { + case a.Kind == groups.OwedBacklog: a.Backlog, a.Answer = backlog(ctx), "the backlog of each seat's stream owed" return a + case a.Kind == groups.OwedAgain && a.Record != "": + a.Answer = o.againRecord(ctx, a.Record) + return a } p, err := o.kept(ctx, a.Of, a.ID) if err != nil { @@ -195,15 +267,9 @@ func (o *owing) answer(ctx context.Context, a groups.OwedAsk, backlog func(conte } var e forgemsg.OwedEnded if a.Kind == groups.OwedDrop { - e, err = o.end(ctx, forgemsg.OwedEnded{Of: a.Of, ID: a.ID, Outcome: forgemsg.OwedDropped, Words: a.Why + " (dropped by " + a.By + ")"}, p.Repository, p.Number) - } else if read, rerr := readerOf(a.Of); rerr != nil { - err = rerr - } else if data, rerr := o.actOf(ctx, a.Of, p, a.ID); rerr != nil { - err = rerr - } else if c, rerr := read(data); rerr != nil { - e, err = o.end(ctx, forgemsg.OwedEnded{Of: a.Of, ID: a.ID, Outcome: forgemsg.OwedUndecodable, Words: rerr.Error()}, p.Repository, p.Number) + e, err = o.end(ctx, forgemsg.OwedEnded{Of: a.Of, ID: a.ID, Outcome: forgemsg.OwedDropped, Words: a.Why + " (dropped by " + a.By + ")"}, callOf(p)) } else { - e, err = o.perform(ctx, c) + e, err = o.again(ctx, a.Of, a.ID, p) } if err != nil { a.Answer = err.Error() @@ -216,6 +282,68 @@ func (o *owing) answer(ctx context.Context, a groups.OwedAsk, backlog func(conte return a } +// callOf is what performed holds of an act as the call an outcome is kept with: its pull request and its record. +func callOf(p Performed) bridge.OwedCall { + return bridge.OwedCall{Repository: p.Repository, Number: p.Number, Record: p.Record} +} + +// again performs one act of of again, read from its seat's stream owed on the pull request p names. +func (o *owing) again(ctx context.Context, of, id string, p Performed) (forgemsg.OwedEnded, error) { + read, err := readerOf(of) + if err != nil { + return forgemsg.OwedEnded{}, err + } + data, err := o.actOf(ctx, of, p, id) + if err != nil { + return forgemsg.OwedEnded{}, err + } + c, err := read(data) + if err != nil { + return o.end(ctx, forgemsg.OwedEnded{Of: of, ID: id, Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, callOf(p)) + } + if c.Record == "" { + c.Record = p.Record // the record kept with the act stays with it + } + return o.perform(ctx, c, nil) +} + +// againRecord performs again every act of the delivery record (its source revision) that performed holds as ended could +// not be performed, oldest first, and answers what each came to; a record none of whose acts so ended is said. +func (o *owing) againRecord(ctx context.Context, record string) string { + keys, err := o.performed.Keys(ctx) + if err != nil { + return "performed cannot be listed, so no act of the record " + record + " is performed again: " + err.Error() + } + var acts []Performed + for _, k := range keys { + v, found, err := o.performed.Get(ctx, k) + var p Performed + if err != nil || !found || json.Unmarshal(v, &p) != nil { + continue + } + if p.Record == record && p.Ended.Outcome == forgemsg.OwedCouldNotBePerformed { + acts = append(acts, p) + } + } + if len(acts) == 0 { + return "the record " + record + " holds no act that ended " + forgemsg.OwedCouldNotBePerformed + } + sort.Slice(acts, func(i, j int) bool { return acts[i].Ended.At.Before(acts[j].Ended.At) }) + lines := make([]string, 0, len(acts)) + for _, p := range acts { + e, err := o.again(ctx, p.Ended.Of, p.Ended.ID, p) + switch { + case err != nil: + lines = append(lines, p.Ended.ID+": "+err.Error()) + case e.Words != "": + lines = append(lines, p.Ended.ID+": "+e.Outcome+": "+e.Words) + default: + lines = append(lines, p.Ended.ID+": "+e.Outcome) + } + } + return fmt.Sprintf("%d acts of the record %s performed again: %s", len(acts), record, strings.Join(lines, "; ")) +} + // asked answers one value of the state owed-asks not yet answered, written back by compare-and-set at its revision; an // answered one (this bridge's own write) is left. func (o *owing) asked(ctx context.Context, kv jetstream.KeyValue, key string, rev uint64, value []byte, backlog func(context.Context) []groups.Backlog) { diff --git a/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go b/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go index 1a4c65481..1d576804a 100644 --- a/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go +++ b/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go @@ -96,6 +96,9 @@ func TestForge(t *testing.T) { {"no id", func(e *OwedEnded) { e.ID = "" }, "no id"}, {"a refusal with no words", func(e *OwedEnded) { e.Outcome = OwedRefused }, "no words"}, {"dropped with no reason", func(e *OwedEnded) { e.Outcome = OwedDropped }, "no words"}, + // Design 47 H2 §807 (H-A-432 (2)): an act whose pull request closed unmerged is said not-needed, naming it. + {"not needed, naming why", func(e *OwedEnded) { e.Outcome, e.Words = OwedNotNeeded, "novox/x#9 closed unmerged" }, ""}, + {"not needed with no words", func(e *OwedEnded) { e.Outcome = OwedNotNeeded }, "no words"}, {"no time", func(e *OwedEnded) { e.At = time.Time{} }, "no time"}, {"undecodable with no id is said", func(e *OwedEnded) { e.ID, e.Outcome, e.Words = "", OwedUndecodable, "not json" }, ""}, } { diff --git a/modules/mesh-delivery-forge-bridge/forgemsg/owed.go b/modules/mesh-delivery-forge-bridge/forgemsg/owed.go index e67e3bbec..4fd508efb 100644 --- a/modules/mesh-delivery-forge-bridge/forgemsg/owed.go +++ b/modules/mesh-delivery-forge-bridge/forgemsg/owed.go @@ -13,18 +13,20 @@ import ( const OwedEndedEvent = "owed.ended" // The outcomes of an owed act (design 47 G §164, §168): performed, refused (the forge's own words), could not be -// performed (the forge did not answer within the call's bound, with the error), undecodable (an act the bridge cannot -// read, with why) and, from the tool `bridge_owed_drop`, dropped. +// performed (the forge did not answer within the bridge's owed-away-bound, naming every try, or answered an error), +// undecodable (an act the bridge cannot read, with why), from the tool `bridge_owed_drop`, dropped, and not-needed (the +// act's pull request closed unmerged, read before a try, so it is not performed; design 47 H2 §807, H-A-432 (2)). const ( OwedPerformed = "performed" OwedRefused = "refused" OwedCouldNotBePerformed = "could not be performed" OwedUndecodable = "undecodable" OwedDropped = "dropped" + OwedNotNeeded = "not-needed" ) // OwedOutcomes are the outcomes an owed.ended says, one of them. -var OwedOutcomes = []string{OwedPerformed, OwedRefused, OwedCouldNotBePerformed, OwedUndecodable, OwedDropped} +var OwedOutcomes = []string{OwedPerformed, OwedRefused, OwedCouldNotBePerformed, OwedUndecodable, OwedDropped, OwedNotNeeded} // OwedEnded is an owed act's outcome (design 47 G §164): the act's id and one outcome, with the words of any outcome // but performed. Of is the seat that said the act (mesh-delivery for the gate's), so it hears its own ids (§164). An act the bridge cannot decode may name no id. diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/owed.go b/modules/mesh-delivery-forge-bridge/internal/bridge/owed.go index 4fc6d2ed4..739aee121 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/owed.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/owed.go @@ -19,8 +19,11 @@ type OwedCall struct { Act string Repository string Number int - Verb string - Input any + // Record is the delivery record the act is for, its source revision (design 47 H2 §807, H-A-432 (1)); empty when the + // owed named none. + Record string + Verb string + Input any } // ErrUndecodable is an owed event the bridge cannot read: it is said as owed.ended `undecodable`, never only logged. @@ -36,7 +39,7 @@ func GateOwed(data []byte) (OwedCall, error) { if err := json.Unmarshal(data, &o); err != nil { return OwedCall{}, undecodable("the gate's owed is not json: %v", err) } - c := OwedCall{Of: seat.Seat, ID: o.ID, Act: o.Act, Repository: o.Repository, Number: int(o.Number)} + c := OwedCall{Of: seat.Seat, ID: o.ID, Act: o.Act, Repository: o.Repository, Number: int(o.Number), Record: o.Record} if err := c.common(o.Head); err != nil { return c, err } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/owed_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/owed_test.go index e59b80e5f..9e3e8b076 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/owed_test.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/owed_test.go @@ -62,6 +62,13 @@ func TestAnOwedEventIsTheSeatGitsVerbAndItsAnswerOneOutcome(t *testing.T) { t.Errorf("%s: %s %+v %v, want %s %+v", c.name, got.Verb, got.Input, err, c.verb, c.input) } } + // Design 47 H2 §807 (H-A-432 (1)): the delivery record an owed act names is carried with it; an act naming none has none. + if got, err := GateOwed(edit(gate("owed is said, a status"), "record", "main.212")); err != nil || got.Record != "main.212" { + t.Errorf("an act of a record: %+v %v, want its record main.212", got, err) + } + if got, err := GateOwed(gate("owed is said, a status")); err != nil || got.Record != "" { + t.Errorf("an act of no record: %+v %v, want none", got, err) + } answers := func(verb string) []json.RawMessage { return gitfixture.All(t, verb+" answers", "verb:"+verb+".answer") } failed := func(verb string) json.RawMessage { return gitfixture.Given(t, verb+" answers an error, never none", "verb:"+verb+".answer") diff --git a/modules/mesh-delivery-forge-bridge/internal/groups/owedask.go b/modules/mesh-delivery-forge-bridge/internal/groups/owedask.go index 02e4d96a4..7f436ba56 100644 --- a/modules/mesh-delivery-forge-bridge/internal/groups/owedask.go +++ b/modules/mesh-delivery-forge-bridge/internal/groups/owedask.go @@ -46,9 +46,12 @@ type Backlog struct { // and once the bridge answered, when, what it came to (the outcome said as owed.ended) and its words; a backlog ask is // answered with the consumers' backlog. type OwedAsk struct { - Kind string `json:"kind"` - Of string `json:"of,omitempty"` - ID string `json:"id,omitempty"` + Kind string `json:"kind"` + Of string `json:"of,omitempty"` + ID string `json:"id,omitempty"` + // Record, for again instead of an act, is a delivery record's source revision: every act of it that ended could not + // be performed is performed again (design 47 H2 §807, H-A-432 (1)). + Record string `json:"record,omitempty"` Why string `json:"why,omitempty"` By string `json:"by,omitempty"` At time.Time `json:"at"` @@ -84,6 +87,20 @@ func NewOwedAsk(kind, of, id, why, by string, at time.Time) (string, []byte, err return key, v, err } +// NewOwedRecordAsk is the key and value of an ask to perform again every act of one delivery record (its source revision, +// .) that ended could not be performed: under again.record and the record (unpadded base64url). It says why. +func NewOwedRecordAsk(record, why, by string, at time.Time) (string, []byte, error) { + a := OwedAsk{Kind: OwedAgain, Record: strings.TrimSpace(record), Why: strings.TrimSpace(why), By: by, At: at.UTC()} + if a.Record == "" { + return "", nil, fmt.Errorf("a record is named by its source revision, .") + } + if a.Why == "" { + return "", nil, fmt.Errorf("performing a record's acts again says why") + } + v, err := json.Marshal(a) + return OwedAgain + ".record." + base64.RawURLEncoding.EncodeToString([]byte(a.Record)), v, err +} + // DecodeOwedAsk reads a value of owed-asks. func DecodeOwedAsk(data []byte) (OwedAsk, error) { var a OwedAsk diff --git a/modules/mesh-delivery-forge-bridge/internal/groups/performed.go b/modules/mesh-delivery-forge-bridge/internal/groups/performed.go index eaaa09fe5..81ea61f66 100644 --- a/modules/mesh-delivery-forge-bridge/internal/groups/performed.go +++ b/modules/mesh-delivery-forge-bridge/internal/groups/performed.go @@ -16,11 +16,14 @@ func PerformedKey(of, id string) string { return of + "." + base64.RawURLEncoding.EncodeToString([]byte(id)) } -// Performed is one value of the state performed (§163, H-A-182): the outcome said as owed.ended, with when, and the +// Performed is one value of the state performed (§163, H-A-182): the outcome said as owed.ended, with when, the // repository and number of the pull request the act names, so bridge_owed reads one pull request's acts and -// bridge_owed_again finds the act on its pull request's subject of the stream owed (H-A-181). +// bridge_owed_again finds the act on its pull request's subject of the stream owed (H-A-181), and the delivery record +// the act is for (its source revision, .), so bridge_owed_again by record finds the record's acts in +// this state (design 47 H2 §807, H-A-432 (1)); absent for an act whose owed named none. type Performed struct { Ended forgemsg.OwedEnded `json:"ended"` Repository string `json:"repository,omitempty"` Number int `json:"number,omitempty"` + Record string `json:"record,omitempty"` } diff --git a/modules/mesh-delivery-forge-bridge/manifest_test.go b/modules/mesh-delivery-forge-bridge/manifest_test.go index 3c81cb8dd..19041986d 100644 --- a/modules/mesh-delivery-forge-bridge/manifest_test.go +++ b/modules/mesh-delivery-forge-bridge/manifest_test.go @@ -26,6 +26,18 @@ type stream struct { type state struct { Name string `json:"name"` History int `json:"history"` + MaxMiB int `json:"max-mib"` +} + +// setting is an entry of config. +type setting struct { + Type string `json:"type"` + Description string `json:"description"` + Trusted bool `json:"trusted"` + Runtime bool `json:"runtime"` + Minimum float64 `json:"minimum"` + Maximum float64 `json:"maximum"` + Default any `json:"default"` } // kept is an entry of requires in its long form: kept data on the bus (novox/hq ADR 0595). @@ -49,19 +61,20 @@ type artifact struct { } type manifest struct { - Module string `json:"module"` - Version string `json:"version"` - Slug string `json:"slug"` - Consumes []string `json:"consumes"` - Streams []stream `json:"streams"` - State []state `json:"state"` - Requires []kept `json:"requires"` - Reads []string `json:"reads"` - Tools []string `json:"tools"` - Invokes []string `json:"invokes"` - OwnSecrets map[string]string `json:"own-secrets"` - SecretsOwner string `json:"secrets-owner"` - Resources []map[string]any `json:"resources"` + Module string `json:"module"` + Version string `json:"version"` + Slug string `json:"slug"` + Consumes []string `json:"consumes"` + Streams []stream `json:"streams"` + State []state `json:"state"` + Requires []kept `json:"requires"` + Reads []string `json:"reads"` + Tools []string `json:"tools"` + Invokes []string `json:"invokes"` + Config map[string]setting `json:"config"` + OwnSecrets map[string]string `json:"own-secrets"` + SecretsOwner string `json:"secrets-owner"` + Resources []map[string]any `json:"resources"` Build struct { Artifacts []artifact `json:"artifacts"` } `json:"build"` @@ -93,6 +106,7 @@ func sorted(s []string) []string { } func TestManifestDeclarations(t *testing.T) { // 1235/AC1 + t.Run("the process", manifestProcess) // design 47 H2 §819: one top-level test fewer m, top := load(t) if m.Module != "mesh-delivery-forge-bridge" { t.Errorf("module = %q", m.Module) @@ -114,8 +128,9 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 if len(m.Reads) != 0 { t.Errorf("reads = %v, want none", m.Reads) } - // No pull, tree, file, open pull requests, merge or group is asked (design 47 H §739, §764). - if want := []string{"seat:git.branch", "seat:git.comment", "seat:git.note", "seat:git.status", "seat:git.tags", "seat:git.view", + // No tree, file, open pull requests, merge or group is asked (design 47 H §739, §764); one pull request's state is read + // before each try of an act (design 47 H2 §807, H-A-432 (2)). + if want := []string{"seat:git.branch", "seat:git.comment", "seat:git.note", "seat:git.pull", "seat:git.status", "seat:git.tags", "seat:git.view", "seat:mesh-broker.show", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source", "seat:mesh-hand-acts.hand-act"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } @@ -125,20 +140,30 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 if m.SecretsOwner != "mesh-delivery-forge-bridge" { t.Errorf("secrets-owner = %q", m.SecretsOwner) } - // The head held for each watched branch (design 47 H §739), and the asks about what is owed. What it performed of what is - // owed is kept data in the new form, one key per act, kept 7 days as the owner's stream owed is; its size is the - // template's own (design 47 *What the gate owes the forge*, ADR 0595 §3). No inputs, end-asks or findings (H §764, H-A-48). - if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "owed-asks", History: 1}}) { + // The head held for each watched branch (design 47 H §739), and the asks about what is owed, each capped at 8 MiB + // (design 47 H2 §812, H-A-411 (2)). What it performed of what is owed is kept data in the new form, one key per act, + // kept 3 days by the bucket's TTL, performed-keep, and capped at 8 MiB, pruned toward it at the bridge's writes (§812, + // H-A-433). No inputs, end-asks or findings (H §764, H-A-48). + if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1, MaxMiB: 8}, {Name: "owed-asks", History: 1, MaxMiB: 8}}) { t.Errorf("state = %+v, want the buckets heads and owed-asks", m.State) } - if want := []kept{{Name: "performed", Storage: "key-value", Template: "state", Options: map[string]any{"history": float64(1), "ttl-seconds": float64(604800)}}}; !reflect.DeepEqual(m.Requires, want) { + if want := []kept{{Name: "performed", Storage: "key-value", Template: "state", Options: map[string]any{"history": float64(1), "ttl-seconds": float64(259200), "max-mib": float64(8)}}}; !reflect.DeepEqual(m.Requires, want) { t.Errorf("requires = %+v, want performed alone as %+v", m.Requires, want) } // bridge_tags reads a package's tags for the builder's floor (design 51); bridge_heads reads the state heads (design 47 - // H-A-85); the owed tools ask the bridge through owed-asks. - if want := []string{"bridge_heads", "bridge_owed", "bridge_owed_again", "bridge_owed_drop", "bridge_tags"}; !reflect.DeepEqual(sorted(m.Tools), want) { + // H-A-85); the owed tools ask the bridge through owed-asks; the two prune tools act on performed (§812, H-A-380 (3)). + if want := []string{"bridge_heads", "bridge_owed", "bridge_owed_again", "bridge_owed_drop", "bridge_tags", + "mesh-delivery-forge-bridge_state_keep", "mesh-delivery-forge-bridge_state_prune_now"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } + // performed's prune starts past prune-mark and ends at prune-target, settings read at run time (§812). + // Each owed act is a kept call at the module's pace until owed-away-bound (§807, §775). + for key, def := range map[string]any{"prune-mark": float64(70), "prune-target": float64(50), "owed-away-bound": "60m", + "kept-call-pace-first": "1s", "kept-call-pace-max": "30s"} { + if c := m.Config[key]; c.Default != def || !c.Runtime { + t.Errorf("config %s = %+v, want a runtime setting defaulting to %v", key, c, def) + } + } // Design 47 H §764 (B3 AC1): the earlier model's names are in no file of the bridge but tests asserting their absence. retired := []string{"pushed", "forge.group.merged", "bridge_groups", "bridge_group_end_again", "bridge_merged_message", "OwedActMerge", "mesh-delivery-checker", "seat:git.merge-pull", "branch-unprotected", "ForgeMerged", "ForgePushed"} @@ -162,7 +187,7 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 } } -func TestManifestProcess(t *testing.T) { // 1235/AC1 +func manifestProcess(t *testing.T) { // 1235/AC1 m, _ := load(t) var procs []map[string]any for _, r := range m.Resources { diff --git a/modules/mesh-delivery-forge-bridge/module.json b/modules/mesh-delivery-forge-bridge/module.json index 2cdf59b7a..0576160e0 100644 --- a/modules/mesh-delivery-forge-bridge/module.json +++ b/modules/mesh-delivery-forge-bridge/module.json @@ -14,20 +14,23 @@ { "name": "forge", "events": ["forge.**", "owed.ended"], "max-age-days": 7 } ], "state": [ - { "name": "heads", "history": 1 }, - { "name": "owed-asks", "history": 1 } + { "name": "heads", "history": 1, "max-mib": 8 }, + { "name": "owed-asks", "history": 1, "max-mib": 8 } ], "tools": [ "bridge_tags", "bridge_heads", "bridge_owed", "bridge_owed_again", - "bridge_owed_drop" + "bridge_owed_drop", + "mesh-delivery-forge-bridge_state_prune_now", + "mesh-delivery-forge-bridge_state_keep" ], "invokes": [ "seat:git.status", "seat:git.comment", "seat:git.view", + "seat:git.pull", "seat:git.note", "seat:mesh-hand-acts.hand-act", "seat:git.tags", @@ -41,9 +44,50 @@ "name": "performed", "storage": "key-value", "template": "state", - "options": { "history": 1, "ttl-seconds": 604800 } + "options": { "history": 1, "ttl-seconds": 259200, "max-mib": 8 } } ], + "config": { + "owed-away-bound": { + "type": "string", + "description": "How long the bridge calls an owed act again through transport faults and the seat git's away before it says owed.ended could not be performed, naming its tries, a Go duration (design 47 H2 §807).", + "trusted": false, + "runtime": true, + "default": "60m" + }, + "kept-call-pace-first": { + "type": "string", + "description": "the first wait before a kept call met by a transport fault is made again, doubled per call up to kept-call-pace-max, a Go duration (design 47 H2 §775).", + "trusted": false, + "runtime": true, + "default": "1s" + }, + "kept-call-pace-max": { + "type": "string", + "description": "the most a kept call's wait between calls grows to, a Go duration (design 47 H2 §775).", + "trusted": false, + "runtime": true, + "default": "30s" + }, + "prune-mark": { + "type": "number", + "description": "The fill of the bridge's state performed, in percent of its cap, past which the bridge prunes it at its own write, oldest acts first (design 47 H2 §812): below the bus module's raise mark of 75 %.", + "trusted": false, + "runtime": true, + "minimum": 1, + "maximum": 100, + "default": 70 + }, + "prune-target": { + "type": "number", + "description": "The fill, in percent of the cap, a prune of the bridge's state performed takes it down to (design 47 H2 §812); below prune-mark.", + "trusted": false, + "runtime": true, + "minimum": 1, + "maximum": 100, + "default": 50 + } + }, "own-secrets": { "broker": "${dir:state}/broker" },