diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks_tools.go new file mode 100644 index 000000000..550bfb4fb --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks_tools.go @@ -0,0 +1,131 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// The seat mesh-wants' work queue read and steered by hand (design 47 H2 §777, the tools table; H-A-410): assigner_asks +// answers what waits on the seat's queue (read by the runtime without taking it) and what the assigner took and keeps +// dropped in its state taken; assigner_ask_drop ends one ask through the seat's act drop, written in the hand-act log first. + +const ( + asksTool = "assigner_asks" + askDropTool = "assigner_ask_drop" +) + +// asksAnswerMost is the most an asks answer carries of the taken keys (§811: at most 768 KB with capped and total). +const asksAnswerMost = 768 << 10 + +// takenReader is what assigner_asks reads of the state taken (stdio.KeptState). +type takenReader interface { + Keys() ([]string, error) + Get(key string) (*stdio.StateEntry, error) +} + +// asksOf is assigner_asks' answer: the asks on the seat's queue, each waiting, waiting again or held, the ids taken (key mesh-wants.) and the drops of +// asks not pulled yet (dropped.), each list cut at the answer's bound with capped and total. +func asksOf(waiting func(seat string) (stdio.Waiting, error), taken takenReader) (map[string]any, error) { + w, err := waiting(meshwants.Seat) + if err != nil { + return nil, fmt.Errorf("the asks waiting on the seat %s's work queue are not read: %w", meshwants.Seat, err) + } + keys, err := taken.Keys() + if err != nil { + return nil, fmt.Errorf("the state %s is not listed: %w", process.TakenState, err) + } + slices.Sort(keys) + // An ask the queue handed out and not acknowledged (again) is held when this holder took it, else it waits again after a + // nak (H-A-410's addenda): the bus says what it knows, the state taken the rest. + size := 0 + waits := make([]map[string]any, 0, len(w.Asks)) + for _, a := range w.Asks { + size += len(a.ID) + len(a.Module) + 64 + state := "waiting" + if a.Again { + state = "waiting again" + if slices.Contains(keys, process.TakenKey(meshwants.Seat, a.ID)) { + state = "held" + } + } + waits = append(waits, map[string]any{"id": a.ID, "module": a.Module, "age_seconds": a.AgeSeconds, "state": state}) + } + takenIDs, dropped := []string{}, []map[string]any{} + total, capped := 0, w.Capped + for _, k := range keys { + if id, ok := strings.CutPrefix(k, process.TakenKey(meshwants.Seat, "")); ok { + total++ + if size += len(id) + 4; size > asksAnswerMost { + capped = true + continue + } + takenIDs = append(takenIDs, id) + continue + } + id, ok := strings.CutPrefix(k, droppedPrefix) + if !ok { + continue + } + total++ + e, err := taken.Get(k) + if err != nil || e == nil { + continue + } + var d askDrop + if json.Unmarshal(e.Value, &d) != nil { + continue + } + if size += len(id) + len(d.By) + len(d.Why) + 64; size > asksAnswerMost { + capped = true + continue + } + dropped = append(dropped, map[string]any{"id": id, "by": d.By, "why": d.Why, "at": d.At}) + } + return map[string]any{"waiting": waits, "taken": takenIDs, "dropped": dropped, "capped": capped, "total": total + len(w.Asks)}, nil +} + +// asksTools are the module's tools of the seat's work queue. +func asksTools(waiting func(string) (stdio.Waiting, error), taken takenReader, ask func(string, any) (json.RawMessage, error)) []stdio.Tool { + return []stdio.Tool{ + {Name: asksTool, Description: "The asks of the seat mesh-wants' work queue (design 47 H2 §777): each one on it (its id, the module " + + "that asked it, its age and whether it waits, waits again after a hand-back or is held here), the ids the assigner took (kept for ask-taken-bound) and the drops of asks not pulled yet, " + + "cut at 768 KB with capped and total. Read-only.", + Input: schema(map[string]string{}, nil), + Run: func(map[string]any) (any, error) { + return asksOf(waiting, taken) + }}, + {Name: askDropTool, Description: "End one ask of the seat mesh-wants by hand (design 47 H2 §777, §778): written in the hand-act log " + + "first, then the seat's act drop ends it: one running is cancelled, one waiting is ended when it is pulled; either ends " + + "its step could not run naming the drop and who asked it.", + Input: schema(map[string]string{"ask": "the ask's id", "why": "why the ask is ended"}, []string{"ask", "why"}), + Run: func(args map[string]any) (any, error) { + id, why := strings.TrimSpace(arg(args, "ask")), strings.TrimSpace(arg(args, "why")) + if id == "" || why == "" { + return nil, errors.New("name the ask and why it is ended") + } + by := stdio.CallerOf(args) + taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: askDropTool + " " + id, Why: why + " (" + by + ")", Cause: "ask-drop"}) + if err != nil { + return nil, fmt.Errorf("the hand act was not recorded, so nothing was dropped: %w", err) + } + raw, err := ask("seat:"+meshwants.Seat+"."+meshwants.VerbDrop, meshwants.DropInput{ID: id, By: askDropTool + " (" + by + ")", Why: why}) + if err != nil { + return nil, err + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("%s answered what is not JSON: %w", meshwants.VerbDrop, err) + } + out["hand-act"] = taken.Log + return out, nil + }}, + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go index 76646e847..60cc21601 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go @@ -638,7 +638,7 @@ func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) { } got := map[string]meshwants.StepEnded{} for _, m := range []string{"a", "b", "d"} { - steps.take(context.Background(), "ask-"+m, step(m, false)) + pullOne(steps, "ask-"+m, step(m, false)) e := <-ends got[e.Module] = e } @@ -649,7 +649,7 @@ func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) { mu.Lock() fixed = true mu.Unlock() - steps.take(context.Background(), "ask-a2", step("a", true)) + pullOne(steps, "ask-a2", step("a", true)) moved := map[string]meshwants.StepEnded{} for range 2 { select { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go index 7f6e6acef..fc282ed30 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go @@ -126,7 +126,7 @@ func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) { from := b.journal.mark() _, stop := b.startStoppable(t) - b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second) + b.journal.await(t, from, "the composition step is pulled from", "the composition step served", 30*time.Second) select { case <-b.seatListRead: case <-time.After(30 * time.Second): diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go index 587861683..04615205f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go @@ -2,14 +2,11 @@ package main import ( "context" - "encoding/json" + "errors" "sync" "time" - "github.com/nats-io/nats.go" - meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" - "git.novox.be/novox/mesh-sdk/go/process" ) // The composition step (design 47 G §22, §354, §63): the assigner takes the seat mesh-wants' accepted kind `step`, the @@ -44,9 +41,18 @@ type compositionSteps struct { logf func(string, ...any) // ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own. ticks func(d time.Duration) (<-chan time.Time, func()) - mu sync.Mutex - // stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave). - stops map[string]context.CancelFunc + // taken is the state taken (stepqueue.go): each ask pulled is taken once by its id, for takenBound; the drops of asks not + // pulled yet are kept there too (H-A-404). + taken takenStore + takenBound time.Duration + // inProgress is how often a running step's pulled ask is said in progress: the seat definition's in-progress (§777). + inProgress func() time.Duration + mu sync.Mutex + // stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave), or the ask is + // dropped (the cause an *askDrop). + stops map[string]context.CancelCauseFunc + // held are the messages pulled from the seat's work queue for each running step's ask, acknowledged once its end is said. + held map[string][]queueMsg } // progressEvery is how often a running composition step says its progress (design 47 G §496, A-262). @@ -92,20 +98,6 @@ func (p *stepPhase) get() string { return p.phase } -// take takes one ask of the accepted kind `step` by its id and body: a step that is no composition or validate step, or an -// ask with no id, is left with a log line; one taken is run in its own goroutine and its end said. -func (c *compositionSteps) take(ctx context.Context, id string, data []byte) bool { - var run meshwants.StepRun - err := json.Unmarshal(data, &run) - how := c.howOf(run.Step, false) - if err != nil || id == "" || how == nil { - c.logf("a step ask that is no composition or validate step with an id is left: %v", err) - return false - } - go c.say1(ctx, id, run, how, "") - return true -} - // howOf is what runs a step, or judges it once taken up at the start: by its step, composition or validate; nil for another. func (c *compositionSteps) howOf(step string, resumed bool) func(context.Context, string, meshwants.StepRun) stepOutcome { switch { @@ -124,16 +116,16 @@ func (c *compositionSteps) howOf(step string, resumed bool) func(context.Context // say1 runs one step with how and says its end, naming in `shared` the runs it read (§723); again is, for an end said again, the // ask of the run whose end makes it say again (sayEnd). A step whose wait was ended by its pipeline's supersede or cancel (leave) // says nothing: the runner ended it already. -func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome, again string) { +func (c *compositionSteps) say1(parent context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome, again string) { started := c.now().UTC() - ctx, stopWait := context.WithCancel(ctx) + ctx, stopWait := context.WithCancelCause(parent) c.mu.Lock() if c.stops == nil { - c.stops = map[string]context.CancelFunc{} + c.stops = map[string]context.CancelCauseFunc{} } if c.stops[id] != nil { // the step asked under id is running already: it says its own end c.mu.Unlock() - stopWait() + stopWait(nil) return } c.stops[id] = stopWait @@ -142,14 +134,25 @@ func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.St c.mu.Lock() delete(c.stops, id) c.mu.Unlock() - stopWait() + stopWait(nil) }() phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch} stop := c.sayProgress(id, run.Asker, phase) + stopInProgress := c.inProgressEvery(id) o := how(context.WithValue(ctx, phaseKey{}, phase), id, run) + stopInProgress() stop() if ctx.Err() != nil { + if parent.Err() != nil { // the assigner stops: its pulled asks are handed again to its next start + return + } + var d *askDrop + if errors.As(context.Cause(ctx), &d) { // dropped through the seat's drop (§778, H-A-401) + c.sayEnd(id, run, &started, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: d.Error(), Shared: o.Shared}, again) + return + } c.logf("the composition step %s of %s says no end: its wait ended (%v)", id, run.Pipeline, context.Cause(ctx)) + c.settle(id, true) // the runner ended it already: its ask is over return } c.sayEnd(id, run, &started, o, again) @@ -175,9 +178,11 @@ func (c *compositionSteps) sayEnd(id string, run meshwants.StepRun, started *tim msgID = id + "." + again } if err := c.say(meshwants.EventStepEnded+"."+c.node, end, msgID); err != nil { - c.logf("saying the end of the composition step %s: %v", id, err) + c.logf("saying the end of the composition step %s: %v; its ask is handed again", id, err) + c.settle(id, false) return } + c.settle(id, true) // the ask's end is said: its pulled messages leave the seat's work queue (§777) if c.ended != nil { c.ended(id, run, ended) } @@ -210,7 +215,7 @@ func (c *compositionSteps) leave(ids []string) { defer c.mu.Unlock() for _, id := range ids { if stop := c.stops[id]; stop != nil { - stop() + stop(nil) } } } @@ -243,21 +248,3 @@ func (c *compositionSteps) sayProgress(id, asker string, phase *stepPhase) (stop }() return func() { close(done); <-ended } } - -// stepAskID is an ask's id as the asker names it: its event id, else its message id. -func stepAskID(m *nats.Msg) string { - if id := m.Header.Get("x-event-id"); id != "" { - return id - } - return m.Header.Get(nats.MsgIdHdr) -} - -// serveCompositionSteps hears the composition step asks on the seat mesh-wants' accept subject. -func serveCompositionSteps(ctx context.Context, conn process.Subscriber, c *compositionSteps) error { - sub, err := conn.Subscribe(meshwants.AcceptSubjectStep, func(m *nats.Msg) { c.take(ctx, stepAskID(m), m.Data) }) - if err != nil { - return err - } - go func() { <-ctx.Done(); _ = sub.Drain() }() - return nil -} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go index 42615f58a..374f91e2e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go @@ -106,7 +106,7 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T) from := b.journal.mark() b.start(t) - b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second) + b.journal.await(t, from, "the composition step is pulled from", "the composition step served", 30*time.Second) b.journal.await(t, from, "n3 is away (machine-away)", "the machine-away of n3 heard", 30*time.Second) select { @@ -114,14 +114,8 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T) case <-time.After(30 * time.Second): t.Fatal("the assigner did not read the seat list within 30 s") } - step, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.3.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: trunkBranch, + putStep(b.ctx, t, b.js, "step-ask-1", meshwants.StepRun{Pipeline: "main.3.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: trunkBranch, Revision: 3, Commit: engineCommit, Fingerprint: "src2:m1"}) - msg := nats.NewMsg(meshwants.AcceptSubjectStep) - msg.Header.Set("x-event-id", "step-ask-1") - msg.Data = step - if err := nc.PublishMsg(msg); err != nil { - t.Fatal(err) - } // The stand-in Go build seat takes the run asks put under the assigner's name, reads each request by the verb input and says // its end with the composer's verdict: n1 composes, n2 does not. @@ -201,6 +195,7 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T) t.Fatalf("the inputs stay kept past their step-ended: %v", keys) } } + queueEmpty(b.ctx, t, b.js) // the step's ask acknowledged at its end (design 47 H2 §777) } // readInput reads a run's input as a build seat does: the seat mesh-wants' verb input answers its digest and chunk count, then its @@ -288,9 +283,37 @@ func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T } t.Cleanup(func() { _ = ends.Unsubscribe() }) subscribed(t, nc) + // Design 47 H2 §777: a validate ask put while the assigner is stopped waits on the seat's work queue, and is taken (ask-taken) + // and ended after its start. + taken := make(chan *nats.Msg, 8) + takens, err := nc.ChanSubscribe(meshwants.SubjectAskTakenAll, taken) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = takens.Unsubscribe() }) + subscribed(t, nc) + putStep(b.ctx, t, b.js, "validate-9", meshwants.StepRun{Pipeline: trunkBranch + ".9", Step: meshwants.StepRunStepValidate, Module: "a", + Branch: trunkBranch, Revision: 9, Commit: commit}) from := b.journal.mark() b.start(t) - b.journal.await(t, from, "the composition step is heard on", "the steps served", 30*time.Second) + b.journal.await(t, from, "the composition step is pulled from", "the steps served", 30*time.Second) + select { + case m := <-taken: + var at meshwants.AskTakenEvent + if err := json.Unmarshal(m.Data, &at); err != nil || at.Ask != "validate-9" || at.Node != "n1" || at.At.IsZero() || m.Subject != meshwants.SubjectAskTaken("validate-9") { + t.Fatalf("ask-taken on %s = %+v, %v; want validate-9 taken on n1", m.Subject, at, err) + } + case <-time.After(30 * time.Second): + t.Fatal("no ask-taken within 30 s of the start for the ask put while the assigner was stopped") + } + select { + case e := <-ended: + if e.ID != "validate-9" || e.Outcome != meshwants.StepEndedOutcomeFailed || e.Said != reasonCycle+": a:x → b:y → a:x" { + t.Errorf("the validate step of a take-in making a cycle ended %+v", e) + } + case <-time.After(30 * time.Second): + t.Fatal("the validate ask put while the assigner was stopped said no end") + } select { case <-b.seatListRead: case <-time.After(30 * time.Second): @@ -306,12 +329,9 @@ func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T id := fmt.Sprintf("validate-%d", revision) step, _ := json.Marshal(meshwants.StepRun{Pipeline: fmt.Sprintf("%s.%d", trunkBranch, revision), Step: meshwants.StepRunStepValidate, Module: module, Branch: trunkBranch, Revision: revision, Commit: commit}) - msg := nats.NewMsg(meshwants.AcceptSubjectStep) - msg.Header.Set("x-event-id", id) - msg.Data = step - if err := nc.PublishMsg(msg); err != nil { - t.Fatal(err) - } + var asked meshwants.StepRun + _ = json.Unmarshal(step, &asked) + putStep(b.ctx, t, b.js, id, asked) if run != nil { run() } @@ -365,10 +385,8 @@ func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T if e := ask(8, "web", validator); e.Outcome != meshwants.StepEndedOutcomePassed { t.Errorf("the validate step of a manifest with no problem ended %+v", e) } - if e := ask(9, "a", nil); e.Outcome != meshwants.StepEndedOutcomeFailed || e.Said != reasonCycle+": a:x → b:y → a:x" { - t.Errorf("the validate step of a take-in making a cycle ended %+v", e) - } if info, err := worker.Info(b.ctx); err != nil || info.NumPending != 0 { t.Errorf("a validator is asked for the cycle: %+v %v", info, err) } + queueEmpty(b.ctx, t, b.js) // every ask acknowledged at its end (§777) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go index ee2541c18..edf0d478f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go @@ -3,15 +3,114 @@ package main import ( "context" "encoding/json" + "errors" "slices" "strings" "sync" "testing" "time" + "github.com/nats-io/nats.go" + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + stdio "git.novox.be/novox/mesh-sdk/go" ) +// fakeMsg is one ask pulled from the seat's work queue, recording how it was settled. +type fakeMsg struct { + data []byte + h nats.Header + mu sync.Mutex + acks, naks, terms, inProgresss int +} + +func (m *fakeMsg) Data() []byte { return m.data } +func (m *fakeMsg) Headers() nats.Header { return m.h } +func (m *fakeMsg) count(n *int) error { m.mu.Lock(); *n++; m.mu.Unlock(); return nil } +func (m *fakeMsg) Ack() error { return m.count(&m.acks) } +func (m *fakeMsg) Nak() error { return m.count(&m.naks) } +func (m *fakeMsg) Term() error { return m.count(&m.terms) } +func (m *fakeMsg) InProgress() error { return m.count(&m.inProgresss) } +func (m *fakeMsg) settled() (acks, naks, terms, progress int) { + m.mu.Lock() + defer m.mu.Unlock() + return m.acks, m.naks, m.terms, m.inProgresss +} + +// fakeTaken is the state taken: its keys, each with its time to live, and a refusal of TakeOnce when refuse is set. +type fakeTaken struct { + mu sync.Mutex + values map[string][]byte + ttls map[string]time.Duration + refuse error +} + +func newFakeTaken() *fakeTaken { + return &fakeTaken{values: map[string][]byte{}, ttls: map[string]time.Duration{}} +} + +func (f *fakeTaken) TakeOnce(seat, ask string, bound time.Duration) (bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.refuse != nil { + return false, f.refuse + } + key := seat + "." + ask + if _, ok := f.values[key]; ok { + return false, nil + } + f.values[key], f.ttls[key] = []byte(`{}`), bound + return true, nil +} + +func (f *fakeTaken) Get(key string) (*stdio.StateEntry, error) { + f.mu.Lock() + defer f.mu.Unlock() + v, ok := f.values[key] + if !ok { + return nil, nil + } + return &stdio.StateEntry{Key: key, Value: v}, nil +} + +func (f *fakeTaken) RenewTTL(key string, value any, ttl time.Duration) (uint64, error) { + raw, err := json.Marshal(value) + f.mu.Lock() + defer f.mu.Unlock() + f.values[key], f.ttls[key] = raw, ttl + return 1, err +} + +func (f *fakeTaken) Delete(key string) error { + f.mu.Lock() + defer f.mu.Unlock() + delete(f.values, key) + return nil +} + +func (f *fakeTaken) Keys() ([]string, error) { + f.mu.Lock() + defer f.mu.Unlock() + var out []string + for k := range f.values { + out = append(out, k) + } + return out, nil +} + +// pullOne hands c one ask pulled from the seat's work queue under id, with a state taken of its own when c has none. +func pullOne(c *compositionSteps, id string, body []byte) *fakeMsg { + if c.taken == nil { + c.taken = newFakeTaken() + } + m := &fakeMsg{data: body, h: nats.Header{}} + if id != "" { + m.h.Set("x-event-id", id) + } + c.pulled(context.Background(), m) + return m +} + // saidEvents keeps what a publisher says. type saidEvents struct { mu sync.Mutex @@ -45,17 +144,20 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) { Shared: []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}}} }} other, _ := json.Marshal(map[string]any{"pipeline": "main.4.m", "step": "build", "module": "m"}) - if c.take(context.Background(), "ask-0", other) { + if m := pullOne(c, "ask-0", other); m.terms != 1 { t.Fatal("a step ask of another step is taken") } body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main", Commit: "c0ffee", Revision: 4, Fingerprint: "fp1", Rerun: true, By: "lead@n1"}) - if c.take(context.Background(), "", body) { + if m := pullOne(c, "", body); m.terms != 1 { t.Fatal("a step ask with no id is taken") } - if !c.take(context.Background(), "ask-1", body) { + if m := pullOne(c, "ask-1", body); m.terms != 0 { t.Fatal("the composition step is not taken") } + if e := <-said.heard; e != meshwants.EventAskTaken+".ask-1" { // said at the pull (H2 §777) + t.Fatalf("said %s first, want ask-taken", e) + } select { case <-said.heard: case <-time.After(5 * time.Second): @@ -73,6 +175,204 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) { len(end.Shared) != 1 || end.Shared[0] != (meshwants.Shared{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}) { // §723, §729 t.Fatalf("step-ended %+v", end) } + // Design 47 H2 §777, §775 (H-A-381, H-A-401, H-A-404): the asks of the seat's work queue, taken once by id, said ask-taken at + // the pull, in progress while they run, acknowledged once their end is said, and ended by the seat's drop. + t.Run("work queue", queueCases) +} + +func queueCases(t *testing.T) { + at := time.Date(2026, 11, 22, 1, 0, 0, 0, time.UTC) + body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepValidate, Module: "m", Branch: "main", Revision: 4}) + type bed struct { + c *compositionSteps + said *saidEvents + taken *fakeTaken + runs chan string + release chan struct{} + tick chan time.Time + } + newBed := func() *bed { + b := &bed{said: newSaidEvents(), taken: newFakeTaken(), runs: make(chan string, 8), release: make(chan struct{}), tick: make(chan time.Time)} + b.said.heard = make(chan string, 16) + run := func(ctx context.Context, id string, _ meshwants.StepRun) stepOutcome { + b.runs <- id + select { + case <-b.release: + case <-ctx.Done(): + } + return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed} + } + b.c = &compositionSteps{node: "n1", say: b.said.say, now: func() time.Time { return at }, logf: t.Logf, taken: b.taken, takenBound: 10 * time.Minute, + validate: run, inProgress: func() time.Duration { return 20 * time.Second }, + ticks: func(d time.Duration) (<-chan time.Time, func()) { + if d == 20*time.Second { + return b.tick, func() {} + } + return make(chan time.Time), func() {} + }} + return b + } + ended := func(t *testing.T, b *bed) meshwants.StepEnded { + t.Helper() + for { + select { + case e := <-b.said.heard: + if e == meshwants.EventStepEnded+".n1" { + b.said.mu.Lock() + defer b.said.mu.Unlock() + return b.said.said[e].(meshwants.StepEnded) + } + case <-time.After(5 * time.Second): + t.Fatal("no step-ended is said") + } + } + } + settledAt := func(t *testing.T, m *fakeMsg, acks int) { + t.Helper() + deadline := time.After(5 * time.Second) + for { + if a, _, _, _ := m.settled(); a == acks { + return + } + select { + case <-deadline: + t.Fatalf("the ask is acknowledged %d times, want %d", m.acks, acks) + case <-time.After(time.Millisecond): + } + } + } + t.Run("taken once, said, in progress, acknowledged at its end", func(t *testing.T) { + b := newBed() + m := pullOne(b.c, "ask-1", body) + if id := <-b.runs; id != "ask-1" { + t.Fatalf("ran %s", id) + } + b.said.mu.Lock() + taken, ok := b.said.said[meshwants.EventAskTaken+".ask-1"].(meshwants.AskTakenEvent) + takenID := b.said.ids[meshwants.EventAskTaken+".ask-1"] + b.said.mu.Unlock() + if !ok || taken.Ask != "ask-1" || taken.Node != "n1" || !taken.At.Equal(at) || takenID != "ask-1.taken" || b.taken.ttls["mesh-wants.ask-1"] != 10*time.Minute { + t.Fatalf("ask-taken %+v under %q, taken %v", taken, takenID, b.taken.ttls) + } + b.tick <- at + b.tick <- at + // handed again while it runs: held for the run, which answers both at its one end + again := pullOne(b.c, "ask-1", body) + if a, n, _, p := m.settled(); a != 0 || n != 0 || p < 1 { + t.Fatalf("before its end the ask is acked %d, nakked %d, in progress %d times", a, n, p) + } + close(b.release) + if e := ended(t, b); e.ID != "ask-1" || e.Outcome != meshwants.StepEndedOutcomePassed { + t.Fatalf("step-ended %+v", e) + } + settledAt(t, m, 1) + settledAt(t, again, 1) + select { + case id := <-b.runs: + t.Fatalf("the ask handed again ran again (%s)", id) + default: + } + }) + t.Run("taken before is run again, said no ask-taken", func(t *testing.T) { + b := newBed() + _, _ = b.taken.TakeOnce(meshwants.Seat, "ask-2", time.Minute) + close(b.release) + m := pullOne(b.c, "ask-2", body) + <-b.runs + ended(t, b) + settledAt(t, m, 1) + b.said.mu.Lock() + _, said := b.said.said[meshwants.EventAskTaken+".ask-2"] + b.said.mu.Unlock() + if said { + t.Fatal("ask-taken is said again for an ask taken before") + } + }) + t.Run("a refused take is handed again", func(t *testing.T) { + b := newBed() + b.taken.refuse = errors.New("no bus") + m := pullOne(b.c, "ask-3", body) + if _, n, _, _ := m.settled(); n != 1 || len(b.runs) != 0 { + t.Fatalf("nakked %d, ran %d", n, len(b.runs)) + } + }) + t.Run("dropped while it runs", func(t *testing.T) { + b := newBed() + m := pullOne(b.c, "ask-4", body) + <-b.runs + if _, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-4"}); err == nil { + t.Fatal("a drop with no why is taken") + } + a, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-4", Why: "its bound passed"}) + if err != nil || a["held"] != true { + t.Fatalf("drop answered %v, %v", a, err) + } + e := ended(t, b) + if e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || e.Said != "the ask was dropped by n1.mesh-pipeline-runner: its bound passed" { + t.Fatalf("step-ended %+v", e) + } + settledAt(t, m, 1) + }) + t.Run("dropped before it was pulled", func(t *testing.T) { + b := newBed() + a, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-5", By: "the runner", Why: "its bound passed"}) + if err != nil || a["held"] != false || b.taken.ttls["dropped.ask-5"] != 10*time.Minute { + t.Fatalf("drop answered %v, %v; taken %v", a, err, b.taken.ttls) + } + m := pullOne(b.c, "ask-5", body) + e := ended(t, b) + if e.ID != "ask-5" || e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || e.Said != "the ask was dropped by the runner: its bound passed" { + t.Fatalf("step-ended %+v", e) + } + settledAt(t, m, 1) + if len(b.runs) != 0 { + t.Fatal("a dropped ask ran") + } + if v, _ := b.taken.Get("dropped.ask-5"); v != nil { + t.Fatal("the drop's key stays once its ask ended") + } + }) + t.Run("the asks tools", func(t *testing.T) { + taken := newFakeTaken() + _, _ = taken.TakeOnce(meshwants.Seat, "ask-6", time.Minute) + _, _ = taken.RenewTTL("dropped.ask-7", askDrop{By: "the runner", Why: "its bound passed", At: at}, time.Minute) + waiting := func(seat string) (stdio.Waiting, error) { + if seat != meshwants.Seat { + t.Errorf("the waiting asks of %s are read", seat) + } + return stdio.Waiting{Asks: []stdio.WaitingAsk{{ID: "ask-8", Module: "mesh-delivery-pipeline", AgeSeconds: 3}, + {ID: "ask-6", Module: "mesh-delivery-pipeline", Again: true}, {ID: "ask-9", Module: "mesh-delivery-pipeline", Again: true}}}, nil + } + var asked []string + ask := func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + asked = append(asked, key+" "+string(raw)) + if strings.HasSuffix(key, "."+meshwants.VerbDrop) { + return json.RawMessage(`{"held":false}`), nil + } + return json.RawMessage(`{}`), nil + } + tools := map[string]stdio.Tool{} + for _, tool := range asksTools(waiting, taken, ask) { + tools[tool.Name] = tool + } + out, err := tools[asksTool].Run(map[string]any{}) + a, _ := out.(map[string]any) + w, _ := a["waiting"].([]map[string]any) + d, _ := a["dropped"].([]map[string]any) + if err != nil || len(w) != 3 || w[0]["id"] != "ask-8" || w[0]["state"] != "waiting" || w[1]["state"] != "held" || w[2]["state"] != "waiting again" || !slices.Equal(a["taken"].([]string), []string{"ask-6"}) || len(d) != 1 || + d[0]["id"] != "ask-7" || d[0]["by"] != "the runner" || a["total"] != 5 || a["capped"] != false { + t.Fatalf("assigner_asks answered %+v, %v", out, err) + } + if _, err := tools[askDropTool].Run(map[string]any{"ask": "ask-8"}); err == nil || len(asked) != 0 { + t.Fatalf("a drop with no why: %v, asked %v", err, asked) + } + out, err = tools[askDropTool].Run(map[string]any{"ask": "ask-8", "why": "stuck"}) + if err != nil || len(asked) != 2 || !strings.HasPrefix(asked[0], "seat:mesh-hand-acts.hand-act ") || !strings.Contains(asked[0], askDropTool+" ask-8") || + !strings.HasPrefix(asked[1], "seat:mesh-wants.drop ") || !strings.Contains(asked[1], `"id":"ask-8"`) || !strings.Contains(asked[1], `"why":"stuck"`) { + t.Fatalf("assigner_ask_drop answered %v, %v; asked %v", out, err, asked) + } + }) } // Design 47 G §496, A-262: a running composition step says step-progress. on every tick of its ticker, with the ask, its @@ -98,9 +398,10 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) { }} // A11/AC1 (design 47 H §752, H-A-88): with no callers in mesh-wants, the progress names the asker the step run carries. body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main", Asker: "mesh-pipeline-runner"}) - if !c.take(context.Background(), "ask-1", body) { + if m := pullOne(c, "ask-1", body); m.terms != 0 { t.Fatal("the composition step is not taken") } + <-said.heard // ask-taken, at the pull (H2 §777) progress := meshwants.EventStepProgress + ".n1" for range 2 { tick <- at @@ -149,7 +450,7 @@ func TestASupersededStepLeavesItsRuns(t *testing.T) { }} body := func(s meshwants.StepRun) []byte { raw, _ := json.Marshal(s); return raw } for _, p := range []string{"main.3.a", "main.3.b"} { - if !c.take(context.Background(), "ask-"+p, body(step(p))) { + if m := pullOne(c, "ask-"+p, body(step(p))); m.terms != 0 { t.Fatal("not taken") } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 1f67ede76..690041718 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -177,7 +177,7 @@ func startBed(t *testing.T, leaveOut string) *bed { continue } cfg := jetstream.KeyValueConfig{Bucket: name, History: 5} - if name == plc.PulsesBucket() || name == buckets.CompositionRuns() { + if name == plc.PulsesBucket() || name == buckets.CompositionRuns() || name == buckets.Taken() { cfg.History, cfg.LimitMarkerTTL = 1, time.Minute // a time to live per key, as the controller makes it (key-ttl) } kv, err := js.CreateKeyValue(ctx, cfg) @@ -206,6 +206,9 @@ func startBed(t *testing.T, leaveOut string) *bed { if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil { t.Fatal(err) } + // The seat mesh-wants' work queue and its one worker, as the broker derives them from the seat's definition (design 47 H2 + // §777); the assigner pulls its step asks from it. + seatQueue(ctx, t, js) // The node-engine seat's stream of `running`, kept last per subject, which the assigner hears the health of what runs from (§520). if _, err := js.CreateStream(ctx, keptStream(nodeengine.StreamRunning, nodeengine.SubjectRunningAll, jetstream.StreamConfig{MaxMsgsPerSubject: 1})); err != nil { t.Fatal(err) @@ -1983,3 +1986,61 @@ func serveSeatList(t *testing.T, nc *nats.Conn) <-chan struct{} { } return read } + +// seatQueue makes the seat mesh-wants' work queue and its one shared worker from the seat's definition, as the broker derives +// them (design 47 H2 §777). +func seatQueue(ctx context.Context, t *testing.T, js jetstream.JetStream) { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "..", "mesh-delivery-contracts", "seats", meshwants.Seat, "seat.json")) + if err != nil { + t.Fatal(err) + } + def, err := seatdef.Parse(raw) + if err != nil { + t.Fatal(err) + } + stream, worker, ok, err := buscompose.SeatQueue(meshwants.Seat, def.Accepts) + if err != nil || !ok { + t.Fatalf("the seat's work queue: %v %v", ok, err) + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream.Name, Subjects: stream.Subjects, Retention: jetstream.WorkQueuePolicy, + MaxAge: time.Duration(stream.MaxAge) * time.Second, MaxMsgs: int64(stream.MaxMsgs), Duplicates: time.Duration(stream.DuplicatesSeconds) * time.Second}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateOrUpdateConsumer(ctx, stream.Name, jetstream.ConsumerConfig{Durable: worker.Name, FilterSubjects: worker.Filters, AckPolicy: jetstream.AckExplicitPolicy, + AckWait: time.Duration(worker.AckWaitSeconds) * time.Second, MaxDeliver: worker.MaxDeliver, MaxAckPending: worker.MaxAckPending}); err != nil { + t.Fatal(err) + } +} + +// putStep puts one step ask on the seat mesh-wants' work queue under id, as the runner puts it (design 47 H2 §777). +func putStep(ctx context.Context, t *testing.T, js jetstream.JetStream, id string, step meshwants.StepRun) { + t.Helper() + msg := nats.NewMsg(meshwants.AcceptSubjectStep) + msg.Header.Set("x-event-id", id) + msg.Header.Set(nats.MsgIdHdr, id) + msg.Data, _ = json.Marshal(step) + if _, err := js.PublishMsg(ctx, msg); err != nil { + t.Fatalf("the ask %s is not put on the seat's work queue: %v", id, err) + } +} + +// queueEmpty fails t unless the seat mesh-wants' worker holds no ask, waiting or unacknowledged (design 47 H2 §777). +func queueEmpty(ctx context.Context, t *testing.T, js jetstream.JetStream) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + c, err := js.Consumer(ctx, buscompose.SeatQueueStreamName(meshwants.Seat), buscompose.SeatQueueWorkerName(meshwants.Seat)) + if err != nil { + t.Fatal(err) + } + info, err := c.Info(ctx) + if err == nil && info.NumPending == 0 && info.NumAckPending == 0 { + return + } + if time.Now().After(deadline) { + t.Fatalf("the seat's worker holds asks after their ends: %+v %v", info, err) + } + time.Sleep(20 * time.Millisecond) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 7c6230ac3..548b74c34 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -14,8 +14,10 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/buscompose" "io" "log" + "maps" "mesh-delivery-assigner/internal/assigngraph" "mesh-delivery-assigner/internal/assignment" "mesh-delivery-assigner/internal/buckets" @@ -23,6 +25,7 @@ import ( "os/signal" "slices" "strings" + "sync/atomic" "syscall" "time" @@ -89,7 +92,7 @@ func ownBuckets() []string { return []string{buckets.Asks(), buckets.Differences(), buckets.Puts(), buckets.Wants(), plc.AssignmentsBucket(), plc.ProposalsBucket(), plc.CandidatesBucket(), plc.PulsesBucket(), assign.FindingsBucket(), assign.PoliciesBucket(), plc.SeatHoldersBucket(), - engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(), buckets.CompositionRuns()} + engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(), buckets.CompositionRuns(), buckets.Taken()} } // engineReadsState is the assigner's state of what each node's engine says it reads (design 47 H-A-104), one key `` @@ -136,6 +139,7 @@ var keptStates = []announce.State{ {Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211) {Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354) {Name: buckets.CompositionRunsState, Readable: true}, // the composition runs, read by the process and its tools (design 47 G §719, A-468c) + {Name: process.TakenState, Readable: true}, // the asks of the seat mesh-wants taken once, and their drops (design 47 H2 §775, §777) } // kvSet is every bucket the assigner opened, by name. @@ -669,8 +673,19 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil { run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, trunk: trunkOf(iv, w.trunkName), newest: iv.newestOf(), watched: iv.watchedOf, assigned: w.assignedOn, assignment: w.assignmentOf, findings: findings, running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf} + // The seat's work queue (design 47 H2 §777): pulled through its one worker, each ask taken once in the state taken. + takenBound, err := askTakenBound() + if err != nil { + return err + } + var ackWait atomic.Int64 // the worker's ack wait, read when it is bound + ackWait.Store(int64(buscompose.SeatQueueDefaultAckWait)) steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume, - validate: run.validate, resumeValidate: run.resumeValidate, ended: runs.stepEnded, now: time.Now, logf: logger.Printf} + validate: run.validate, resumeValidate: run.resumeValidate, ended: runs.stepEnded, now: time.Now, logf: logger.Printf, + takenBound: takenBound, inProgress: inProgressOf(seatlist.Definition(gate.Reader, meshwants.Seat), func() time.Duration { return time.Duration(ackWait.Load()) })} + if e, ok := bus.(emitting); ok { + steps.taken = e.State(process.TakenState) + } move := steps.move(ctx) // the readers a rerun carried (§727, A-470d) runs.moved.Store(&move) // The state's watch wakes the steps waiting on a run and hears a bound's expiry; the runs standing at the start are taken up @@ -685,9 +700,11 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, _, err := meshhandactsclient.Recorder{Ask: bus.Call}.Record(q) return err } - stopVerbs, err := e.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), compositionVerbs(runs, steps, record)) + verbs := compositionVerbs(runs, steps, record) + maps.Copy(verbs, dropVerb(steps)) // the seat's act drop (§778, H-A-404) + stopVerbs, err := e.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), verbs) if err != nil { - logger.Printf("the verbs %s and %s are not answered until the assigner starts again: %v", meshwants.VerbCompositions, meshwants.VerbCompositionsDrop, err) + logger.Printf("the verbs %s, %s and %s are not answered until the assigner starts again: %v", meshwants.VerbCompositions, meshwants.VerbCompositionsDrop, meshwants.VerbDrop, err) } else { go func() { <-ctx.Done(); stopVerbs() }() } @@ -700,10 +717,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } p.ends.left.Store(&left) } - if err := serveCompositionSteps(ctx, nc, steps); err != nil { - logger.Printf("the composition step is not heard, so no composition step runs until the assigner starts again: %v", err) - } else { - logger.Printf("the composition step is heard on %s", meshwants.AcceptSubjectStep) + if steps.taken == nil { + logger.Printf("the state %s is not bound, so no composition step is pulled until the assigner starts again", process.TakenState) + } else if err := serveStepQueue(ctx, nc, js, steps, func(d time.Duration) { ackWait.Store(int64(d)) }); err != nil { + logger.Printf("the composition step is not pulled, so no composition step runs until the assigner starts again: %v", err) } } if bus != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/stepqueue.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/stepqueue.go new file mode 100644 index 000000000..0b9fdf309 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/stepqueue.go @@ -0,0 +1,318 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// **The seat mesh-wants takes its asks from a work queue and says ask-taken** (design 47 H2 §777, §775, §778; ADR 0525 §2, +// §6; ADR 0668 §2; H-A-381, H-A-401, H-A-404): the broker derives the seat's stream and its one shared worker from the seat's +// definition; the assigner pulls every ask waiting at once, takes each once by its id in its state taken (key +// ., for ask-taken-bound), says ask-taken at the pull, sends in-progress while the step runs and acknowledges an +// ask once it said the ask's end. An ask put while the assigner is down waits on the stream. The seat's act drop ends an ask +// its caller no longer waits for: one running here is cancelled, one not pulled yet is kept in taken as dropped. for +// ask-taken-bound, and either ends its step could not run naming the drop and who asked it. + +// askTakenBoundSetting is the setting the taken key's time to live is (H-A-381 (2)): the holder never receives the +// caller's bound, so the largest caller's, the runner's ask-unanswered-bound, is its default. +const ( + askTakenBoundSetting = "ask-taken-bound" + defaultAskTakenBound = 10 * time.Minute +) + +// droppedPrefix keys a drop of an ask not pulled yet in the state taken (H-A-404). +const droppedPrefix = "dropped." + +// takenStore is the state taken (process.State): an ask taken once by its id, and the drops of asks not pulled yet. +type takenStore interface { + TakeOnce(seat, ask string, bound time.Duration) (bool, error) + Get(key string) (*stdio.StateEntry, error) + RenewTTL(key string, value any, ttl time.Duration) (uint64, error) + Delete(key string) error +} + +// queueMsg is what the steps read of a message pulled from the seat's work queue (jetstream.Msg). +type queueMsg interface { + Data() []byte + Headers() nats.Header + Ack() error + Nak() error + Term() error + InProgress() error +} + +// askDrop is one drop of an ask: who asked it, why and when; the cause of a running step's end. +type askDrop struct { + By string `json:"by,omitempty"` + Why string `json:"why"` + At time.Time `json:"at"` +} + +func (d *askDrop) Error() string { + by := d.By + if by == "" { + by = "its asker" + } + return fmt.Sprintf("the ask was dropped by %s: %s", by, d.Why) +} + +// askIDOf is the ask's id a put carries: its event id, else its message id. +func askIDOf(h nats.Header) string { + if id := h.Get("x-event-id"); id != "" { + return id + } + return h.Get(nats.MsgIdHdr) +} + +// pulled handles one ask pulled from the seat's work queue: an ask that is no composition or validate step with an id is +// terminated, said; one dropped before it was pulled ends could not run naming the drop; one running here already (handed +// again while it runs) is held for that run, which acknowledges it at its end; any other is taken once by its id (ask-taken +// said the first time) and run, in progress while it runs. A repeat of an id taken before that runs here no longer is run +// again against the runs as they stand (H-A-381 (1), H-A-401): its composition runs are joined, never run twice. +func (c *compositionSteps) pulled(ctx context.Context, m queueMsg) { + id := askIDOf(m.Headers()) + var run meshwants.StepRun + err := json.Unmarshal(m.Data(), &run) + how := c.howOf(run.Step, false) + if err != nil || id == "" || how == nil { + c.logf("a step ask on the seat %s's work queue that is no composition or validate step with an id is ended: %v", meshwants.Seat, err) + _ = m.Term() + return + } + if d := c.droppedBefore(id); d != nil { + c.hold(id, m) + c.logf("the ask %s (%s) was dropped before it was pulled: %v", id, run.Pipeline, d) + c.sayEnd(id, run, nil, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: d.Error()}, "") + if err := c.taken.Delete(droppedPrefix + id); err != nil { + c.logf("the drop of the ask %s cannot be deleted from the state %s: %v", id, process.TakenState, err) + } + return + } + if c.running(id) { + c.hold(id, m) + c.logf("the ask %s (%s) was handed again while it runs here: its run answers it", id, run.Pipeline) + return + } + first, err := c.taken.TakeOnce(meshwants.Seat, id, c.takenBound) + if err != nil { + c.logf("the ask %s (%s) is not taken, the state %s refused it: %v; it is handed again", id, run.Pipeline, process.TakenState, err) + _ = m.Nak() + return + } + c.hold(id, m) + if first { + at := c.now().UTC() + if err := c.say(meshwants.EventAskTaken+"."+id, meshwants.AskTakenEvent{Ask: id, Node: c.node, At: at}, id+".taken"); err != nil { + c.logf("saying ask-taken of %s: %v", id, err) + } + } else { + c.logf("the ask %s (%s) was taken before: it is run again against its composition runs as they stand", id, run.Pipeline) + } + go c.say1(ctx, id, run, how, "") +} + +// hold keeps m for the step asked under id: acknowledged once its end is said, in progress while it runs. +func (c *compositionSteps) hold(id string, m queueMsg) { + c.mu.Lock() + defer c.mu.Unlock() + if c.held == nil { + c.held = map[string][]queueMsg{} + } + c.held[id] = append(c.held[id], m) +} + +// settle acknowledges (ack) or hands again (nak) every message held for id. +func (c *compositionSteps) settle(id string, ack bool) { + c.mu.Lock() + ms := c.held[id] + delete(c.held, id) + c.mu.Unlock() + for _, m := range ms { + if ack { + _ = m.Ack() + } else { + _ = m.Nak() + } + } +} + +// inProgressEvery says every message held for id in progress every d, until the returned stop is called. +func (c *compositionSteps) inProgressEvery(id string) (stop func()) { + every := time.Duration(0) + if c.inProgress != nil { + every = c.inProgress() + } + if every <= 0 { + return func() {} + } + ticks := c.ticks + if ticks == nil { + ticks = func(d time.Duration) (<-chan time.Time, func()) { t := time.NewTicker(d); return t.C, t.Stop } + } + tick, stopTicks := ticks(every) + done, ended := make(chan struct{}), make(chan struct{}) + go func() { + defer close(ended) + defer stopTicks() + for { + select { + case <-done: + return + case <-tick: + c.mu.Lock() + ms := append([]queueMsg(nil), c.held[id]...) + c.mu.Unlock() + for _, m := range ms { + _ = m.InProgress() + } + } + } + }() + return func() { close(done); <-ended } +} + +// droppedBefore is the drop kept for an ask not pulled yet, nil for none. +func (c *compositionSteps) droppedBefore(id string) *askDrop { + if c.taken == nil { + return nil + } + e, err := c.taken.Get(droppedPrefix + id) + if err != nil { + c.logf("whether the ask %s was dropped cannot be read from the state %s: %v; it is taken", id, process.TakenState, err) + return nil + } + if e == nil { + return nil + } + var d askDrop + if json.Unmarshal(e.Value, &d) != nil { + return nil + } + return &d +} + +// dropAsk is the seat's act drop (§778, H-A-404): a step running here for the ask is cancelled with the drop as its cause and +// ends could not run naming it; one not running here is kept in taken as dropped. for ask-taken-bound, so its pull ends +// it the same way. It answers whether the ask was held here. +func (c *compositionSteps) dropAsk(caller string, in meshwants.DropInput) (meshwants.DropAnswer, error) { + id, why := strings.TrimSpace(in.ID), strings.TrimSpace(in.Why) + if id == "" || why == "" { + return nil, errors.New("give the ask's id and why it is dropped") + } + by := strings.TrimSpace(in.By) + if by == "" { + by = caller + } + d := &askDrop{By: by, Why: why, At: c.now().UTC()} + c.mu.Lock() + stop := c.stops[id] + c.mu.Unlock() + if stop != nil { + stop(d) + c.logf("the ask %s running here is dropped: %v", id, d) + return meshwants.DropAnswer{"held": true}, nil + } + if c.taken == nil { + return nil, fmt.Errorf("the state %s is not bound, so the drop of %s cannot be kept", process.TakenState, id) + } + if _, err := c.taken.RenewTTL(droppedPrefix+id, d, c.takenBound); err != nil { + return nil, fmt.Errorf("writing the drop of %s in the state %s: %w", id, process.TakenState, err) + } + c.logf("the ask %s, not running here, is kept dropped until it is pulled: %v", id, d) + return meshwants.DropAnswer{"held": false}, nil +} + +// dropVerb is the process's handler of the seat mesh-wants' act drop. +func dropVerb(c *compositionSteps) map[string]process.SeatHandler { + return map[string]process.SeatHandler{ + meshwants.VerbDrop: func(caller string, raw json.RawMessage) (any, error) { + var in meshwants.DropInput + if err := json.Unmarshal(raw, &in); err != nil { + return nil, fmt.Errorf("the arguments are not JSON: %w", err) + } + return c.dropAsk(caller, in) + }, + } +} + +// inProgressOf is the seat's in-progress as its definition names it, once the seat list is read; before, a third of the +// worker's ack wait, which the broker derived from the same definition (60 s and 20 s, §777; H-A-401). +func inProgressOf(def func() (seatdef.Definition, bool), ackWait func() time.Duration) func() time.Duration { + return func() time.Duration { + if d, ok := def(); ok { + if _, worker, ok, err := buscompose.SeatQueue(meshwants.Seat, d.Accepts); err == nil && ok && worker.InProgressSeconds > 0 { + return time.Duration(worker.InProgressSeconds) * time.Second + } + } + return ackWait() / 3 + } +} + +// serveStepQueue pulls the seat's work queue through its one shared worker, every ask waiting at once (at most the worker's +// max ack pending), once the broker made it: a worker not there yet is said, and bound when the bus says it was created. +func serveStepQueue(ctx context.Context, conn process.Subscriber, js jetstream.JetStream, c *compositionSteps, ackWait func(time.Duration)) error { + stream, worker := buscompose.SeatQueueStreamName(meshwants.Seat), buscompose.SeatQueueWorkerName(meshwants.Seat) + var once sync.Once + bind := func() bool { + cons, err := js.Consumer(ctx, stream, worker) + if err != nil { + return false + } + bound := false + once.Do(func() { + bound = true + info := cons.CachedInfo() + if info != nil && info.Config.AckWait > 0 { + ackWait(info.Config.AckWait) + } + pending := 1000 + if info != nil && info.Config.MaxAckPending > 0 { + pending = info.Config.MaxAckPending + } + cc, err := cons.Consume(func(m jetstream.Msg) { c.pulled(ctx, m) }, jetstream.PullMaxMessages(pending)) + if err != nil { + c.logf("the seat %s's work queue %s is not pulled: %v", meshwants.Seat, worker, err) + return + } + c.logf("the composition step is pulled from %s on %s", worker, stream) + go func() { <-ctx.Done(); cc.Stop() }() + }) + return bound + } + if bind() { + return nil + } + c.logf("the seat %s's work queue %s on %s is not there yet: its asks wait on it, and it is pulled when the bus says it was created", meshwants.Seat, worker, stream) + sub, err := conn.Subscribe("$JS.EVENT.ADVISORY.CONSUMER.CREATED."+stream+"."+worker, func(*nats.Msg) { go bind() }) + if err != nil { + return fmt.Errorf("hearing the creation of the seat %s's work queue: %w", meshwants.Seat, err) + } + go func() { <-ctx.Done(); _ = sub.Unsubscribe() }() + if bind() { // made between the first look and the subscription + _ = sub.Unsubscribe() + } + return nil +} + +// askTakenBound reads the setting ask-taken-bound (§781). +func askTakenBound() (time.Duration, error) { + d, err := stdio.SettingDuration(askTakenBoundSetting, defaultAskTakenBound) + if err != nil { + return 0, errors.New("the assigner's " + err.Error()) + } + return d, nil +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go index c71e7dbd5..94437bcb3 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go @@ -22,6 +22,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/seatlist" "git.novox.be/novox/mesh-sdk/go/stateobjects" "mesh-delivery-assigner/internal/buckets" @@ -416,6 +417,7 @@ func serveTools() error { tools = append(tools, whyTools(runtimeStore{})...) tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...) tools = append(tools, wantDropTools(runtimeStore{}, stdio.Ask)...) + tools = append(tools, asksTools(stdio.QueueWaiting, stdio.State(process.TakenState), stdio.Ask)...) // the seat's work queue (§777) return stdio.Serve("", tools) } diff --git a/modules/mesh-delivery-assigner/internal/buckets/buckets.go b/modules/mesh-delivery-assigner/internal/buckets/buckets.go index a894baafb..a3aacb739 100644 --- a/modules/mesh-delivery-assigner/internal/buckets/buckets.go +++ b/modules/mesh-delivery-assigner/internal/buckets/buckets.go @@ -44,6 +44,10 @@ func Inputs() string { return own(InputsState) } func CompositionRuns() string { return own(CompositionRunsState) } func Policies() string { return own(PoliciesState) } +// Taken is the bucket of the state taken (design 47 H2 §775, §777): the asks of the seat mesh-wants taken once, and their +// drops; named as mesh-sdk's process.TakenState. +func Taken() string { return own("taken") } + // AsksHistory is the revisions the bucket of asks keeps per key. const AsksHistory = 5 diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 8a3db0c22..b7bc88695 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -70,7 +70,7 @@ func TestManifest(t *testing.T) { } if want := map[string]int{"asks": 5, "differences": 64, "puts": 20, "wants": 20, "assignments": 64, "placement": 64, "copies": 1, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5, views.MarksState: 1, views.StatusState: 1, - "engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "inputs": 1, "composition-runs": 1}; !reflect.DeepEqual(states, want) { + "engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "inputs": 1, "composition-runs": 1, "taken": 1}; !reflect.DeepEqual(states, want) { t.Errorf("state = %v, want %v", states, want) } if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 { @@ -93,7 +93,7 @@ func TestManifest(t *testing.T) { // revision, entry and difference from its revisions kept as objects in the old section, form 1, capped at 512 MiB. if len(m.Claims) != 2 || m.Claims[0].Name != meshmoduleassigner.Seat || m.Claims[0].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[0].Serves, []string{"assignments", "proposals", "propose", "withdraw", "again", "placement"}) || - m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input", "compositions", "compositions-drop"}) { + m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input", "compositions", "compositions-drop", "drop"}) { t.Errorf("the assigner claims %+v, want mesh-module-assigner with its five verbs and mesh-wants with its five", m.Claims) } if len(m.Objects) != 1 || m.Objects[0].Name != "assignment-revisions" || m.Objects[0].MaxMiB != 512 || m.Objects[0].Form != 1 { @@ -105,7 +105,7 @@ func TestManifest(t *testing.T) { // which lists and asks again what each node's engine reads. if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_waits", "assigner_inputs", "assigner_inputs_drop", "assigner_compositions", "assigner_compositions_drop", "assignments", "seat-holders", "wants", "asks", "assigner_read_assignments", "assigner_read_seat_holders", "assigner_read_wants", "assigner_read_findings", "assigner_read_differences", "assigner_read_copies", - "assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_want_drop"}) { + "assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_want_drop", "assigner_asks", "assigner_ask_drop"}) { t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads, the seat list's two (ADR 0636 §7) assigner_ask_again (ADR 0668 §5) and assigner_why (design 47 G A-439)", m.Tools) } if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs { @@ -172,9 +172,11 @@ func TestManifest(t *testing.T) { } file, _ := os.ReadFile("module.json") if err := json.Unmarshal(file, &raw); err != nil || raw.Config["outside-wait-minutes"].Default != float64(30) || raw.Config["outside-settle-seconds"].Default != float64(90) || - raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 10 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) || + raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 13 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) || raw.Config["composition-runs-cap-mib"].Default != float64(16) || raw.Config["composition-runs-kept"].Default != "24h" || - raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) { + raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) || + // design 47 H2 §775, H-A-381 (2): the taken key's bound and the kept-call pace, runtime settings + raw.Config["ask-taken-bound"].Default != "10m" || raw.Config["kept-call-pace-first"].Default != "1s" || raw.Config["kept-call-pace-max"].Default != "30s" { t.Errorf("config = %+v (%v), want outside-wait-minutes of 30, outside-settle-seconds of 90 and ask-operator-for of take-off,remove-node,stops (A-425a)", raw.Config, err) } for _, f := range []struct{ env, path, key string }{ diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 329791381..e1502c7d3 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -27,7 +27,8 @@ "difference", "input", "compositions", - "compositions-drop" + "compositions-drop", + "drop" ] } ], @@ -129,7 +130,9 @@ "assigner_ask_again", "assigner_written", "assigner_why", - "assigner_want_drop" + "assigner_want_drop", + "assigner_asks", + "assigner_ask_drop" ], "state": [ { "name": "assignments", "history": 64 }, @@ -152,7 +155,8 @@ { "name": "waits", "history": 5 }, { "name": "put-waits", "history": 1 }, { "name": "inputs", "history": 1, "max-mib": 64 }, - { "name": "composition-runs", "history": 1, "key-ttl": true, "max-mib": 64 } + { "name": "composition-runs", "history": 1, "key-ttl": true, "max-mib": 64 }, + { "name": "taken", "history": 1, "key-ttl": true, "max-mib": 8 } ], "config": { "outside-wait-minutes": { @@ -226,6 +230,27 @@ "maximum": 1000, "trusted": false, "default": 20 + }, + "ask-taken-bound": { + "type": "string", + "description": "how long the assigner keeps the id of an ask it took from the seat mesh-wants' work queue, so a repeat of it is taken once, and the drop of an ask not pulled yet, a Go duration such as 10m (design 47 H2 §775, §778; H-A-381 (2), H-A-404).", + "trusted": false, + "default": "10m", + "runtime": true + }, + "kept-call-pace-first": { + "type": "string", + "description": "the first wait before a kept call met by a transport fault is made again, doubled per call up to kept-call-pace-max, a Go duration (design 47 H2 §775).", + "trusted": false, + "default": "1s", + "runtime": true + }, + "kept-call-pace-max": { + "type": "string", + "description": "the most a kept call's wait between calls grows to, a Go duration (design 47 H2 §775).", + "trusted": false, + "default": "30s", + "runtime": true } }, "objects": [