From dc72860ce0a766364d822ffb295666308d76c19b Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:05:07 +0100 Subject: [PATCH 01/12] A2: the assigner pulls the seat mesh-wants' work queue and takes each ask once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A step ask put while the assigner was down waited for nobody: the plain subscription heard only what came while it ran. The assigner now pulls every waiting ask from the seat's one worker, takes it once by its id in its state taken (for ask-taken-bound), says ask-taken at the pull, sends in-progress while the step runs and acknowledges once the step's end is said. It serves the seat's act drop: a running ask is cancelled, one not yet pulled is kept in taken as dropped.; either ends its step could not run naming the drop. assigner_asks reads what waits on the queue (stdio.QueueWaiting) and what is taken or dropped; assigner_ask_drop ends one ask, written in the hand-act log. The manifest declares taken, drop, the two tools, ask-taken-bound and the kept-call pace as runtime settings. Builds: design 47 H2 §777, §775, §778, §820; H-A-381, H-A-401, H-A-404, H-A-410 --- .../cmd/mesh-delivery-assigner/asks_tools.go | 131 ++++++++ .../compositionrun_test.go | 4 +- .../compositionruns_integration_test.go | 2 +- .../mesh-delivery-assigner/compositionstep.go | 79 ++--- .../compositionstep_integration_test.go | 54 ++- .../compositionstep_test.go | 311 ++++++++++++++++- .../integration_test.go | 63 +++- .../cmd/mesh-delivery-assigner/main.go | 33 +- .../cmd/mesh-delivery-assigner/stepqueue.go | 318 ++++++++++++++++++ .../cmd/mesh-delivery-assigner/tools.go | 2 + .../internal/buckets/buckets.go | 4 + .../mesh-delivery-assigner/manifest_test.go | 12 +- modules/mesh-delivery-assigner/module.json | 31 +- 13 files changed, 955 insertions(+), 89 deletions(-) create mode 100644 modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks_tools.go create mode 100644 modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/stepqueue.go diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks_tools.go new file mode 100644 index 000000000..550bfb4fb --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks_tools.go @@ -0,0 +1,131 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// The seat mesh-wants' work queue read and steered by hand (design 47 H2 §777, the tools table; H-A-410): assigner_asks +// answers what waits on the seat's queue (read by the runtime without taking it) and what the assigner took and keeps +// dropped in its state taken; assigner_ask_drop ends one ask through the seat's act drop, written in the hand-act log first. + +const ( + asksTool = "assigner_asks" + askDropTool = "assigner_ask_drop" +) + +// asksAnswerMost is the most an asks answer carries of the taken keys (§811: at most 768 KB with capped and total). +const asksAnswerMost = 768 << 10 + +// takenReader is what assigner_asks reads of the state taken (stdio.KeptState). +type takenReader interface { + Keys() ([]string, error) + Get(key string) (*stdio.StateEntry, error) +} + +// asksOf is assigner_asks' answer: the asks on the seat's queue, each waiting, waiting again or held, the ids taken (key mesh-wants.) and the drops of +// asks not pulled yet (dropped.), each list cut at the answer's bound with capped and total. +func asksOf(waiting func(seat string) (stdio.Waiting, error), taken takenReader) (map[string]any, error) { + w, err := waiting(meshwants.Seat) + if err != nil { + return nil, fmt.Errorf("the asks waiting on the seat %s's work queue are not read: %w", meshwants.Seat, err) + } + keys, err := taken.Keys() + if err != nil { + return nil, fmt.Errorf("the state %s is not listed: %w", process.TakenState, err) + } + slices.Sort(keys) + // An ask the queue handed out and not acknowledged (again) is held when this holder took it, else it waits again after a + // nak (H-A-410's addenda): the bus says what it knows, the state taken the rest. + size := 0 + waits := make([]map[string]any, 0, len(w.Asks)) + for _, a := range w.Asks { + size += len(a.ID) + len(a.Module) + 64 + state := "waiting" + if a.Again { + state = "waiting again" + if slices.Contains(keys, process.TakenKey(meshwants.Seat, a.ID)) { + state = "held" + } + } + waits = append(waits, map[string]any{"id": a.ID, "module": a.Module, "age_seconds": a.AgeSeconds, "state": state}) + } + takenIDs, dropped := []string{}, []map[string]any{} + total, capped := 0, w.Capped + for _, k := range keys { + if id, ok := strings.CutPrefix(k, process.TakenKey(meshwants.Seat, "")); ok { + total++ + if size += len(id) + 4; size > asksAnswerMost { + capped = true + continue + } + takenIDs = append(takenIDs, id) + continue + } + id, ok := strings.CutPrefix(k, droppedPrefix) + if !ok { + continue + } + total++ + e, err := taken.Get(k) + if err != nil || e == nil { + continue + } + var d askDrop + if json.Unmarshal(e.Value, &d) != nil { + continue + } + if size += len(id) + len(d.By) + len(d.Why) + 64; size > asksAnswerMost { + capped = true + continue + } + dropped = append(dropped, map[string]any{"id": id, "by": d.By, "why": d.Why, "at": d.At}) + } + return map[string]any{"waiting": waits, "taken": takenIDs, "dropped": dropped, "capped": capped, "total": total + len(w.Asks)}, nil +} + +// asksTools are the module's tools of the seat's work queue. +func asksTools(waiting func(string) (stdio.Waiting, error), taken takenReader, ask func(string, any) (json.RawMessage, error)) []stdio.Tool { + return []stdio.Tool{ + {Name: asksTool, Description: "The asks of the seat mesh-wants' work queue (design 47 H2 §777): each one on it (its id, the module " + + "that asked it, its age and whether it waits, waits again after a hand-back or is held here), the ids the assigner took (kept for ask-taken-bound) and the drops of asks not pulled yet, " + + "cut at 768 KB with capped and total. Read-only.", + Input: schema(map[string]string{}, nil), + Run: func(map[string]any) (any, error) { + return asksOf(waiting, taken) + }}, + {Name: askDropTool, Description: "End one ask of the seat mesh-wants by hand (design 47 H2 §777, §778): written in the hand-act log " + + "first, then the seat's act drop ends it: one running is cancelled, one waiting is ended when it is pulled; either ends " + + "its step could not run naming the drop and who asked it.", + Input: schema(map[string]string{"ask": "the ask's id", "why": "why the ask is ended"}, []string{"ask", "why"}), + Run: func(args map[string]any) (any, error) { + id, why := strings.TrimSpace(arg(args, "ask")), strings.TrimSpace(arg(args, "why")) + if id == "" || why == "" { + return nil, errors.New("name the ask and why it is ended") + } + by := stdio.CallerOf(args) + taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: askDropTool + " " + id, Why: why + " (" + by + ")", Cause: "ask-drop"}) + if err != nil { + return nil, fmt.Errorf("the hand act was not recorded, so nothing was dropped: %w", err) + } + raw, err := ask("seat:"+meshwants.Seat+"."+meshwants.VerbDrop, meshwants.DropInput{ID: id, By: askDropTool + " (" + by + ")", Why: why}) + if err != nil { + return nil, err + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("%s answered what is not JSON: %w", meshwants.VerbDrop, err) + } + out["hand-act"] = taken.Log + return out, nil + }}, + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go index 76646e847..60cc21601 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go @@ -638,7 +638,7 @@ func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) { } got := map[string]meshwants.StepEnded{} for _, m := range []string{"a", "b", "d"} { - steps.take(context.Background(), "ask-"+m, step(m, false)) + pullOne(steps, "ask-"+m, step(m, false)) e := <-ends got[e.Module] = e } @@ -649,7 +649,7 @@ func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) { mu.Lock() fixed = true mu.Unlock() - steps.take(context.Background(), "ask-a2", step("a", true)) + pullOne(steps, "ask-a2", step("a", true)) moved := map[string]meshwants.StepEnded{} for range 2 { select { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go index 7f6e6acef..fc282ed30 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionruns_integration_test.go @@ -126,7 +126,7 @@ func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) { from := b.journal.mark() _, stop := b.startStoppable(t) - b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second) + b.journal.await(t, from, "the composition step is pulled from", "the composition step served", 30*time.Second) select { case <-b.seatListRead: case <-time.After(30 * time.Second): diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go index 587861683..04615205f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go @@ -2,14 +2,11 @@ package main import ( "context" - "encoding/json" + "errors" "sync" "time" - "github.com/nats-io/nats.go" - meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" - "git.novox.be/novox/mesh-sdk/go/process" ) // The composition step (design 47 G §22, §354, §63): the assigner takes the seat mesh-wants' accepted kind `step`, the @@ -44,9 +41,18 @@ type compositionSteps struct { logf func(string, ...any) // ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own. ticks func(d time.Duration) (<-chan time.Time, func()) - mu sync.Mutex - // stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave). - stops map[string]context.CancelFunc + // taken is the state taken (stepqueue.go): each ask pulled is taken once by its id, for takenBound; the drops of asks not + // pulled yet are kept there too (H-A-404). + taken takenStore + takenBound time.Duration + // inProgress is how often a running step's pulled ask is said in progress: the seat definition's in-progress (§777). + inProgress func() time.Duration + mu sync.Mutex + // stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave), or the ask is + // dropped (the cause an *askDrop). + stops map[string]context.CancelCauseFunc + // held are the messages pulled from the seat's work queue for each running step's ask, acknowledged once its end is said. + held map[string][]queueMsg } // progressEvery is how often a running composition step says its progress (design 47 G §496, A-262). @@ -92,20 +98,6 @@ func (p *stepPhase) get() string { return p.phase } -// take takes one ask of the accepted kind `step` by its id and body: a step that is no composition or validate step, or an -// ask with no id, is left with a log line; one taken is run in its own goroutine and its end said. -func (c *compositionSteps) take(ctx context.Context, id string, data []byte) bool { - var run meshwants.StepRun - err := json.Unmarshal(data, &run) - how := c.howOf(run.Step, false) - if err != nil || id == "" || how == nil { - c.logf("a step ask that is no composition or validate step with an id is left: %v", err) - return false - } - go c.say1(ctx, id, run, how, "") - return true -} - // howOf is what runs a step, or judges it once taken up at the start: by its step, composition or validate; nil for another. func (c *compositionSteps) howOf(step string, resumed bool) func(context.Context, string, meshwants.StepRun) stepOutcome { switch { @@ -124,16 +116,16 @@ func (c *compositionSteps) howOf(step string, resumed bool) func(context.Context // say1 runs one step with how and says its end, naming in `shared` the runs it read (§723); again is, for an end said again, the // ask of the run whose end makes it say again (sayEnd). A step whose wait was ended by its pipeline's supersede or cancel (leave) // says nothing: the runner ended it already. -func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome, again string) { +func (c *compositionSteps) say1(parent context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome, again string) { started := c.now().UTC() - ctx, stopWait := context.WithCancel(ctx) + ctx, stopWait := context.WithCancelCause(parent) c.mu.Lock() if c.stops == nil { - c.stops = map[string]context.CancelFunc{} + c.stops = map[string]context.CancelCauseFunc{} } if c.stops[id] != nil { // the step asked under id is running already: it says its own end c.mu.Unlock() - stopWait() + stopWait(nil) return } c.stops[id] = stopWait @@ -142,14 +134,25 @@ func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.St c.mu.Lock() delete(c.stops, id) c.mu.Unlock() - stopWait() + stopWait(nil) }() phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch} stop := c.sayProgress(id, run.Asker, phase) + stopInProgress := c.inProgressEvery(id) o := how(context.WithValue(ctx, phaseKey{}, phase), id, run) + stopInProgress() stop() if ctx.Err() != nil { + if parent.Err() != nil { // the assigner stops: its pulled asks are handed again to its next start + return + } + var d *askDrop + if errors.As(context.Cause(ctx), &d) { // dropped through the seat's drop (§778, H-A-401) + c.sayEnd(id, run, &started, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: d.Error(), Shared: o.Shared}, again) + return + } c.logf("the composition step %s of %s says no end: its wait ended (%v)", id, run.Pipeline, context.Cause(ctx)) + c.settle(id, true) // the runner ended it already: its ask is over return } c.sayEnd(id, run, &started, o, again) @@ -175,9 +178,11 @@ func (c *compositionSteps) sayEnd(id string, run meshwants.StepRun, started *tim msgID = id + "." + again } if err := c.say(meshwants.EventStepEnded+"."+c.node, end, msgID); err != nil { - c.logf("saying the end of the composition step %s: %v", id, err) + c.logf("saying the end of the composition step %s: %v; its ask is handed again", id, err) + c.settle(id, false) return } + c.settle(id, true) // the ask's end is said: its pulled messages leave the seat's work queue (§777) if c.ended != nil { c.ended(id, run, ended) } @@ -210,7 +215,7 @@ func (c *compositionSteps) leave(ids []string) { defer c.mu.Unlock() for _, id := range ids { if stop := c.stops[id]; stop != nil { - stop() + stop(nil) } } } @@ -243,21 +248,3 @@ func (c *compositionSteps) sayProgress(id, asker string, phase *stepPhase) (stop }() return func() { close(done); <-ended } } - -// stepAskID is an ask's id as the asker names it: its event id, else its message id. -func stepAskID(m *nats.Msg) string { - if id := m.Header.Get("x-event-id"); id != "" { - return id - } - return m.Header.Get(nats.MsgIdHdr) -} - -// serveCompositionSteps hears the composition step asks on the seat mesh-wants' accept subject. -func serveCompositionSteps(ctx context.Context, conn process.Subscriber, c *compositionSteps) error { - sub, err := conn.Subscribe(meshwants.AcceptSubjectStep, func(m *nats.Msg) { c.take(ctx, stepAskID(m), m.Data) }) - if err != nil { - return err - } - go func() { <-ctx.Done(); _ = sub.Drain() }() - return nil -} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go index 42615f58a..374f91e2e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go @@ -106,7 +106,7 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T) from := b.journal.mark() b.start(t) - b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second) + b.journal.await(t, from, "the composition step is pulled from", "the composition step served", 30*time.Second) b.journal.await(t, from, "n3 is away (machine-away)", "the machine-away of n3 heard", 30*time.Second) select { @@ -114,14 +114,8 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T) case <-time.After(30 * time.Second): t.Fatal("the assigner did not read the seat list within 30 s") } - step, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.3.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: trunkBranch, + putStep(b.ctx, t, b.js, "step-ask-1", meshwants.StepRun{Pipeline: "main.3.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: trunkBranch, Revision: 3, Commit: engineCommit, Fingerprint: "src2:m1"}) - msg := nats.NewMsg(meshwants.AcceptSubjectStep) - msg.Header.Set("x-event-id", "step-ask-1") - msg.Data = step - if err := nc.PublishMsg(msg); err != nil { - t.Fatal(err) - } // The stand-in Go build seat takes the run asks put under the assigner's name, reads each request by the verb input and says // its end with the composer's verdict: n1 composes, n2 does not. @@ -201,6 +195,7 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T) t.Fatalf("the inputs stay kept past their step-ended: %v", keys) } } + queueEmpty(b.ctx, t, b.js) // the step's ask acknowledged at its end (design 47 H2 §777) } // readInput reads a run's input as a build seat does: the seat mesh-wants' verb input answers its digest and chunk count, then its @@ -288,9 +283,37 @@ func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T } t.Cleanup(func() { _ = ends.Unsubscribe() }) subscribed(t, nc) + // Design 47 H2 §777: a validate ask put while the assigner is stopped waits on the seat's work queue, and is taken (ask-taken) + // and ended after its start. + taken := make(chan *nats.Msg, 8) + takens, err := nc.ChanSubscribe(meshwants.SubjectAskTakenAll, taken) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = takens.Unsubscribe() }) + subscribed(t, nc) + putStep(b.ctx, t, b.js, "validate-9", meshwants.StepRun{Pipeline: trunkBranch + ".9", Step: meshwants.StepRunStepValidate, Module: "a", + Branch: trunkBranch, Revision: 9, Commit: commit}) from := b.journal.mark() b.start(t) - b.journal.await(t, from, "the composition step is heard on", "the steps served", 30*time.Second) + b.journal.await(t, from, "the composition step is pulled from", "the steps served", 30*time.Second) + select { + case m := <-taken: + var at meshwants.AskTakenEvent + if err := json.Unmarshal(m.Data, &at); err != nil || at.Ask != "validate-9" || at.Node != "n1" || at.At.IsZero() || m.Subject != meshwants.SubjectAskTaken("validate-9") { + t.Fatalf("ask-taken on %s = %+v, %v; want validate-9 taken on n1", m.Subject, at, err) + } + case <-time.After(30 * time.Second): + t.Fatal("no ask-taken within 30 s of the start for the ask put while the assigner was stopped") + } + select { + case e := <-ended: + if e.ID != "validate-9" || e.Outcome != meshwants.StepEndedOutcomeFailed || e.Said != reasonCycle+": a:x → b:y → a:x" { + t.Errorf("the validate step of a take-in making a cycle ended %+v", e) + } + case <-time.After(30 * time.Second): + t.Fatal("the validate ask put while the assigner was stopped said no end") + } select { case <-b.seatListRead: case <-time.After(30 * time.Second): @@ -306,12 +329,9 @@ func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T id := fmt.Sprintf("validate-%d", revision) step, _ := json.Marshal(meshwants.StepRun{Pipeline: fmt.Sprintf("%s.%d", trunkBranch, revision), Step: meshwants.StepRunStepValidate, Module: module, Branch: trunkBranch, Revision: revision, Commit: commit}) - msg := nats.NewMsg(meshwants.AcceptSubjectStep) - msg.Header.Set("x-event-id", id) - msg.Data = step - if err := nc.PublishMsg(msg); err != nil { - t.Fatal(err) - } + var asked meshwants.StepRun + _ = json.Unmarshal(step, &asked) + putStep(b.ctx, t, b.js, id, asked) if run != nil { run() } @@ -365,10 +385,8 @@ func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T if e := ask(8, "web", validator); e.Outcome != meshwants.StepEndedOutcomePassed { t.Errorf("the validate step of a manifest with no problem ended %+v", e) } - if e := ask(9, "a", nil); e.Outcome != meshwants.StepEndedOutcomeFailed || e.Said != reasonCycle+": a:x → b:y → a:x" { - t.Errorf("the validate step of a take-in making a cycle ended %+v", e) - } if info, err := worker.Info(b.ctx); err != nil || info.NumPending != 0 { t.Errorf("a validator is asked for the cycle: %+v %v", info, err) } + queueEmpty(b.ctx, t, b.js) // every ask acknowledged at its end (§777) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go index ee2541c18..edf0d478f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go @@ -3,15 +3,114 @@ package main import ( "context" "encoding/json" + "errors" "slices" "strings" "sync" "testing" "time" + "github.com/nats-io/nats.go" + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + stdio "git.novox.be/novox/mesh-sdk/go" ) +// fakeMsg is one ask pulled from the seat's work queue, recording how it was settled. +type fakeMsg struct { + data []byte + h nats.Header + mu sync.Mutex + acks, naks, terms, inProgresss int +} + +func (m *fakeMsg) Data() []byte { return m.data } +func (m *fakeMsg) Headers() nats.Header { return m.h } +func (m *fakeMsg) count(n *int) error { m.mu.Lock(); *n++; m.mu.Unlock(); return nil } +func (m *fakeMsg) Ack() error { return m.count(&m.acks) } +func (m *fakeMsg) Nak() error { return m.count(&m.naks) } +func (m *fakeMsg) Term() error { return m.count(&m.terms) } +func (m *fakeMsg) InProgress() error { return m.count(&m.inProgresss) } +func (m *fakeMsg) settled() (acks, naks, terms, progress int) { + m.mu.Lock() + defer m.mu.Unlock() + return m.acks, m.naks, m.terms, m.inProgresss +} + +// fakeTaken is the state taken: its keys, each with its time to live, and a refusal of TakeOnce when refuse is set. +type fakeTaken struct { + mu sync.Mutex + values map[string][]byte + ttls map[string]time.Duration + refuse error +} + +func newFakeTaken() *fakeTaken { + return &fakeTaken{values: map[string][]byte{}, ttls: map[string]time.Duration{}} +} + +func (f *fakeTaken) TakeOnce(seat, ask string, bound time.Duration) (bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.refuse != nil { + return false, f.refuse + } + key := seat + "." + ask + if _, ok := f.values[key]; ok { + return false, nil + } + f.values[key], f.ttls[key] = []byte(`{}`), bound + return true, nil +} + +func (f *fakeTaken) Get(key string) (*stdio.StateEntry, error) { + f.mu.Lock() + defer f.mu.Unlock() + v, ok := f.values[key] + if !ok { + return nil, nil + } + return &stdio.StateEntry{Key: key, Value: v}, nil +} + +func (f *fakeTaken) RenewTTL(key string, value any, ttl time.Duration) (uint64, error) { + raw, err := json.Marshal(value) + f.mu.Lock() + defer f.mu.Unlock() + f.values[key], f.ttls[key] = raw, ttl + return 1, err +} + +func (f *fakeTaken) Delete(key string) error { + f.mu.Lock() + defer f.mu.Unlock() + delete(f.values, key) + return nil +} + +func (f *fakeTaken) Keys() ([]string, error) { + f.mu.Lock() + defer f.mu.Unlock() + var out []string + for k := range f.values { + out = append(out, k) + } + return out, nil +} + +// pullOne hands c one ask pulled from the seat's work queue under id, with a state taken of its own when c has none. +func pullOne(c *compositionSteps, id string, body []byte) *fakeMsg { + if c.taken == nil { + c.taken = newFakeTaken() + } + m := &fakeMsg{data: body, h: nats.Header{}} + if id != "" { + m.h.Set("x-event-id", id) + } + c.pulled(context.Background(), m) + return m +} + // saidEvents keeps what a publisher says. type saidEvents struct { mu sync.Mutex @@ -45,17 +144,20 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) { Shared: []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}}} }} other, _ := json.Marshal(map[string]any{"pipeline": "main.4.m", "step": "build", "module": "m"}) - if c.take(context.Background(), "ask-0", other) { + if m := pullOne(c, "ask-0", other); m.terms != 1 { t.Fatal("a step ask of another step is taken") } body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main", Commit: "c0ffee", Revision: 4, Fingerprint: "fp1", Rerun: true, By: "lead@n1"}) - if c.take(context.Background(), "", body) { + if m := pullOne(c, "", body); m.terms != 1 { t.Fatal("a step ask with no id is taken") } - if !c.take(context.Background(), "ask-1", body) { + if m := pullOne(c, "ask-1", body); m.terms != 0 { t.Fatal("the composition step is not taken") } + if e := <-said.heard; e != meshwants.EventAskTaken+".ask-1" { // said at the pull (H2 §777) + t.Fatalf("said %s first, want ask-taken", e) + } select { case <-said.heard: case <-time.After(5 * time.Second): @@ -73,6 +175,204 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) { len(end.Shared) != 1 || end.Shared[0] != (meshwants.Shared{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}) { // §723, §729 t.Fatalf("step-ended %+v", end) } + // Design 47 H2 §777, §775 (H-A-381, H-A-401, H-A-404): the asks of the seat's work queue, taken once by id, said ask-taken at + // the pull, in progress while they run, acknowledged once their end is said, and ended by the seat's drop. + t.Run("work queue", queueCases) +} + +func queueCases(t *testing.T) { + at := time.Date(2026, 11, 22, 1, 0, 0, 0, time.UTC) + body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepValidate, Module: "m", Branch: "main", Revision: 4}) + type bed struct { + c *compositionSteps + said *saidEvents + taken *fakeTaken + runs chan string + release chan struct{} + tick chan time.Time + } + newBed := func() *bed { + b := &bed{said: newSaidEvents(), taken: newFakeTaken(), runs: make(chan string, 8), release: make(chan struct{}), tick: make(chan time.Time)} + b.said.heard = make(chan string, 16) + run := func(ctx context.Context, id string, _ meshwants.StepRun) stepOutcome { + b.runs <- id + select { + case <-b.release: + case <-ctx.Done(): + } + return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed} + } + b.c = &compositionSteps{node: "n1", say: b.said.say, now: func() time.Time { return at }, logf: t.Logf, taken: b.taken, takenBound: 10 * time.Minute, + validate: run, inProgress: func() time.Duration { return 20 * time.Second }, + ticks: func(d time.Duration) (<-chan time.Time, func()) { + if d == 20*time.Second { + return b.tick, func() {} + } + return make(chan time.Time), func() {} + }} + return b + } + ended := func(t *testing.T, b *bed) meshwants.StepEnded { + t.Helper() + for { + select { + case e := <-b.said.heard: + if e == meshwants.EventStepEnded+".n1" { + b.said.mu.Lock() + defer b.said.mu.Unlock() + return b.said.said[e].(meshwants.StepEnded) + } + case <-time.After(5 * time.Second): + t.Fatal("no step-ended is said") + } + } + } + settledAt := func(t *testing.T, m *fakeMsg, acks int) { + t.Helper() + deadline := time.After(5 * time.Second) + for { + if a, _, _, _ := m.settled(); a == acks { + return + } + select { + case <-deadline: + t.Fatalf("the ask is acknowledged %d times, want %d", m.acks, acks) + case <-time.After(time.Millisecond): + } + } + } + t.Run("taken once, said, in progress, acknowledged at its end", func(t *testing.T) { + b := newBed() + m := pullOne(b.c, "ask-1", body) + if id := <-b.runs; id != "ask-1" { + t.Fatalf("ran %s", id) + } + b.said.mu.Lock() + taken, ok := b.said.said[meshwants.EventAskTaken+".ask-1"].(meshwants.AskTakenEvent) + takenID := b.said.ids[meshwants.EventAskTaken+".ask-1"] + b.said.mu.Unlock() + if !ok || taken.Ask != "ask-1" || taken.Node != "n1" || !taken.At.Equal(at) || takenID != "ask-1.taken" || b.taken.ttls["mesh-wants.ask-1"] != 10*time.Minute { + t.Fatalf("ask-taken %+v under %q, taken %v", taken, takenID, b.taken.ttls) + } + b.tick <- at + b.tick <- at + // handed again while it runs: held for the run, which answers both at its one end + again := pullOne(b.c, "ask-1", body) + if a, n, _, p := m.settled(); a != 0 || n != 0 || p < 1 { + t.Fatalf("before its end the ask is acked %d, nakked %d, in progress %d times", a, n, p) + } + close(b.release) + if e := ended(t, b); e.ID != "ask-1" || e.Outcome != meshwants.StepEndedOutcomePassed { + t.Fatalf("step-ended %+v", e) + } + settledAt(t, m, 1) + settledAt(t, again, 1) + select { + case id := <-b.runs: + t.Fatalf("the ask handed again ran again (%s)", id) + default: + } + }) + t.Run("taken before is run again, said no ask-taken", func(t *testing.T) { + b := newBed() + _, _ = b.taken.TakeOnce(meshwants.Seat, "ask-2", time.Minute) + close(b.release) + m := pullOne(b.c, "ask-2", body) + <-b.runs + ended(t, b) + settledAt(t, m, 1) + b.said.mu.Lock() + _, said := b.said.said[meshwants.EventAskTaken+".ask-2"] + b.said.mu.Unlock() + if said { + t.Fatal("ask-taken is said again for an ask taken before") + } + }) + t.Run("a refused take is handed again", func(t *testing.T) { + b := newBed() + b.taken.refuse = errors.New("no bus") + m := pullOne(b.c, "ask-3", body) + if _, n, _, _ := m.settled(); n != 1 || len(b.runs) != 0 { + t.Fatalf("nakked %d, ran %d", n, len(b.runs)) + } + }) + t.Run("dropped while it runs", func(t *testing.T) { + b := newBed() + m := pullOne(b.c, "ask-4", body) + <-b.runs + if _, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-4"}); err == nil { + t.Fatal("a drop with no why is taken") + } + a, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-4", Why: "its bound passed"}) + if err != nil || a["held"] != true { + t.Fatalf("drop answered %v, %v", a, err) + } + e := ended(t, b) + if e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || e.Said != "the ask was dropped by n1.mesh-pipeline-runner: its bound passed" { + t.Fatalf("step-ended %+v", e) + } + settledAt(t, m, 1) + }) + t.Run("dropped before it was pulled", func(t *testing.T) { + b := newBed() + a, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-5", By: "the runner", Why: "its bound passed"}) + if err != nil || a["held"] != false || b.taken.ttls["dropped.ask-5"] != 10*time.Minute { + t.Fatalf("drop answered %v, %v; taken %v", a, err, b.taken.ttls) + } + m := pullOne(b.c, "ask-5", body) + e := ended(t, b) + if e.ID != "ask-5" || e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || e.Said != "the ask was dropped by the runner: its bound passed" { + t.Fatalf("step-ended %+v", e) + } + settledAt(t, m, 1) + if len(b.runs) != 0 { + t.Fatal("a dropped ask ran") + } + if v, _ := b.taken.Get("dropped.ask-5"); v != nil { + t.Fatal("the drop's key stays once its ask ended") + } + }) + t.Run("the asks tools", func(t *testing.T) { + taken := newFakeTaken() + _, _ = taken.TakeOnce(meshwants.Seat, "ask-6", time.Minute) + _, _ = taken.RenewTTL("dropped.ask-7", askDrop{By: "the runner", Why: "its bound passed", At: at}, time.Minute) + waiting := func(seat string) (stdio.Waiting, error) { + if seat != meshwants.Seat { + t.Errorf("the waiting asks of %s are read", seat) + } + return stdio.Waiting{Asks: []stdio.WaitingAsk{{ID: "ask-8", Module: "mesh-delivery-pipeline", AgeSeconds: 3}, + {ID: "ask-6", Module: "mesh-delivery-pipeline", Again: true}, {ID: "ask-9", Module: "mesh-delivery-pipeline", Again: true}}}, nil + } + var asked []string + ask := func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + asked = append(asked, key+" "+string(raw)) + if strings.HasSuffix(key, "."+meshwants.VerbDrop) { + return json.RawMessage(`{"held":false}`), nil + } + return json.RawMessage(`{}`), nil + } + tools := map[string]stdio.Tool{} + for _, tool := range asksTools(waiting, taken, ask) { + tools[tool.Name] = tool + } + out, err := tools[asksTool].Run(map[string]any{}) + a, _ := out.(map[string]any) + w, _ := a["waiting"].([]map[string]any) + d, _ := a["dropped"].([]map[string]any) + if err != nil || len(w) != 3 || w[0]["id"] != "ask-8" || w[0]["state"] != "waiting" || w[1]["state"] != "held" || w[2]["state"] != "waiting again" || !slices.Equal(a["taken"].([]string), []string{"ask-6"}) || len(d) != 1 || + d[0]["id"] != "ask-7" || d[0]["by"] != "the runner" || a["total"] != 5 || a["capped"] != false { + t.Fatalf("assigner_asks answered %+v, %v", out, err) + } + if _, err := tools[askDropTool].Run(map[string]any{"ask": "ask-8"}); err == nil || len(asked) != 0 { + t.Fatalf("a drop with no why: %v, asked %v", err, asked) + } + out, err = tools[askDropTool].Run(map[string]any{"ask": "ask-8", "why": "stuck"}) + if err != nil || len(asked) != 2 || !strings.HasPrefix(asked[0], "seat:mesh-hand-acts.hand-act ") || !strings.Contains(asked[0], askDropTool+" ask-8") || + !strings.HasPrefix(asked[1], "seat:mesh-wants.drop ") || !strings.Contains(asked[1], `"id":"ask-8"`) || !strings.Contains(asked[1], `"why":"stuck"`) { + t.Fatalf("assigner_ask_drop answered %v, %v; asked %v", out, err, asked) + } + }) } // Design 47 G §496, A-262: a running composition step says step-progress. on every tick of its ticker, with the ask, its @@ -98,9 +398,10 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) { }} // A11/AC1 (design 47 H §752, H-A-88): with no callers in mesh-wants, the progress names the asker the step run carries. body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main", Asker: "mesh-pipeline-runner"}) - if !c.take(context.Background(), "ask-1", body) { + if m := pullOne(c, "ask-1", body); m.terms != 0 { t.Fatal("the composition step is not taken") } + <-said.heard // ask-taken, at the pull (H2 §777) progress := meshwants.EventStepProgress + ".n1" for range 2 { tick <- at @@ -149,7 +450,7 @@ func TestASupersededStepLeavesItsRuns(t *testing.T) { }} body := func(s meshwants.StepRun) []byte { raw, _ := json.Marshal(s); return raw } for _, p := range []string{"main.3.a", "main.3.b"} { - if !c.take(context.Background(), "ask-"+p, body(step(p))) { + if m := pullOne(c, "ask-"+p, body(step(p))); m.terms != 0 { t.Fatal("not taken") } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 1f67ede76..690041718 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -177,7 +177,7 @@ func startBed(t *testing.T, leaveOut string) *bed { continue } cfg := jetstream.KeyValueConfig{Bucket: name, History: 5} - if name == plc.PulsesBucket() || name == buckets.CompositionRuns() { + if name == plc.PulsesBucket() || name == buckets.CompositionRuns() || name == buckets.Taken() { cfg.History, cfg.LimitMarkerTTL = 1, time.Minute // a time to live per key, as the controller makes it (key-ttl) } kv, err := js.CreateKeyValue(ctx, cfg) @@ -206,6 +206,9 @@ func startBed(t *testing.T, leaveOut string) *bed { if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil { t.Fatal(err) } + // The seat mesh-wants' work queue and its one worker, as the broker derives them from the seat's definition (design 47 H2 + // §777); the assigner pulls its step asks from it. + seatQueue(ctx, t, js) // The node-engine seat's stream of `running`, kept last per subject, which the assigner hears the health of what runs from (§520). if _, err := js.CreateStream(ctx, keptStream(nodeengine.StreamRunning, nodeengine.SubjectRunningAll, jetstream.StreamConfig{MaxMsgsPerSubject: 1})); err != nil { t.Fatal(err) @@ -1983,3 +1986,61 @@ func serveSeatList(t *testing.T, nc *nats.Conn) <-chan struct{} { } return read } + +// seatQueue makes the seat mesh-wants' work queue and its one shared worker from the seat's definition, as the broker derives +// them (design 47 H2 §777). +func seatQueue(ctx context.Context, t *testing.T, js jetstream.JetStream) { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "..", "mesh-delivery-contracts", "seats", meshwants.Seat, "seat.json")) + if err != nil { + t.Fatal(err) + } + def, err := seatdef.Parse(raw) + if err != nil { + t.Fatal(err) + } + stream, worker, ok, err := buscompose.SeatQueue(meshwants.Seat, def.Accepts) + if err != nil || !ok { + t.Fatalf("the seat's work queue: %v %v", ok, err) + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream.Name, Subjects: stream.Subjects, Retention: jetstream.WorkQueuePolicy, + MaxAge: time.Duration(stream.MaxAge) * time.Second, MaxMsgs: int64(stream.MaxMsgs), Duplicates: time.Duration(stream.DuplicatesSeconds) * time.Second}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateOrUpdateConsumer(ctx, stream.Name, jetstream.ConsumerConfig{Durable: worker.Name, FilterSubjects: worker.Filters, AckPolicy: jetstream.AckExplicitPolicy, + AckWait: time.Duration(worker.AckWaitSeconds) * time.Second, MaxDeliver: worker.MaxDeliver, MaxAckPending: worker.MaxAckPending}); err != nil { + t.Fatal(err) + } +} + +// putStep puts one step ask on the seat mesh-wants' work queue under id, as the runner puts it (design 47 H2 §777). +func putStep(ctx context.Context, t *testing.T, js jetstream.JetStream, id string, step meshwants.StepRun) { + t.Helper() + msg := nats.NewMsg(meshwants.AcceptSubjectStep) + msg.Header.Set("x-event-id", id) + msg.Header.Set(nats.MsgIdHdr, id) + msg.Data, _ = json.Marshal(step) + if _, err := js.PublishMsg(ctx, msg); err != nil { + t.Fatalf("the ask %s is not put on the seat's work queue: %v", id, err) + } +} + +// queueEmpty fails t unless the seat mesh-wants' worker holds no ask, waiting or unacknowledged (design 47 H2 §777). +func queueEmpty(ctx context.Context, t *testing.T, js jetstream.JetStream) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + c, err := js.Consumer(ctx, buscompose.SeatQueueStreamName(meshwants.Seat), buscompose.SeatQueueWorkerName(meshwants.Seat)) + if err != nil { + t.Fatal(err) + } + info, err := c.Info(ctx) + if err == nil && info.NumPending == 0 && info.NumAckPending == 0 { + return + } + if time.Now().After(deadline) { + t.Fatalf("the seat's worker holds asks after their ends: %+v %v", info, err) + } + time.Sleep(20 * time.Millisecond) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 7c6230ac3..548b74c34 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -14,8 +14,10 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/buscompose" "io" "log" + "maps" "mesh-delivery-assigner/internal/assigngraph" "mesh-delivery-assigner/internal/assignment" "mesh-delivery-assigner/internal/buckets" @@ -23,6 +25,7 @@ import ( "os/signal" "slices" "strings" + "sync/atomic" "syscall" "time" @@ -89,7 +92,7 @@ func ownBuckets() []string { return []string{buckets.Asks(), buckets.Differences(), buckets.Puts(), buckets.Wants(), plc.AssignmentsBucket(), plc.ProposalsBucket(), plc.CandidatesBucket(), plc.PulsesBucket(), assign.FindingsBucket(), assign.PoliciesBucket(), plc.SeatHoldersBucket(), - engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(), buckets.CompositionRuns()} + engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(), buckets.CompositionRuns(), buckets.Taken()} } // engineReadsState is the assigner's state of what each node's engine says it reads (design 47 H-A-104), one key `` @@ -136,6 +139,7 @@ var keptStates = []announce.State{ {Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211) {Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354) {Name: buckets.CompositionRunsState, Readable: true}, // the composition runs, read by the process and its tools (design 47 G §719, A-468c) + {Name: process.TakenState, Readable: true}, // the asks of the seat mesh-wants taken once, and their drops (design 47 H2 §775, §777) } // kvSet is every bucket the assigner opened, by name. @@ -669,8 +673,19 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil { run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, trunk: trunkOf(iv, w.trunkName), newest: iv.newestOf(), watched: iv.watchedOf, assigned: w.assignedOn, assignment: w.assignmentOf, findings: findings, running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf} + // The seat's work queue (design 47 H2 §777): pulled through its one worker, each ask taken once in the state taken. + takenBound, err := askTakenBound() + if err != nil { + return err + } + var ackWait atomic.Int64 // the worker's ack wait, read when it is bound + ackWait.Store(int64(buscompose.SeatQueueDefaultAckWait)) steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume, - validate: run.validate, resumeValidate: run.resumeValidate, ended: runs.stepEnded, now: time.Now, logf: logger.Printf} + validate: run.validate, resumeValidate: run.resumeValidate, ended: runs.stepEnded, now: time.Now, logf: logger.Printf, + takenBound: takenBound, inProgress: inProgressOf(seatlist.Definition(gate.Reader, meshwants.Seat), func() time.Duration { return time.Duration(ackWait.Load()) })} + if e, ok := bus.(emitting); ok { + steps.taken = e.State(process.TakenState) + } move := steps.move(ctx) // the readers a rerun carried (§727, A-470d) runs.moved.Store(&move) // The state's watch wakes the steps waiting on a run and hears a bound's expiry; the runs standing at the start are taken up @@ -685,9 +700,11 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, _, err := meshhandactsclient.Recorder{Ask: bus.Call}.Record(q) return err } - stopVerbs, err := e.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), compositionVerbs(runs, steps, record)) + verbs := compositionVerbs(runs, steps, record) + maps.Copy(verbs, dropVerb(steps)) // the seat's act drop (§778, H-A-404) + stopVerbs, err := e.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), verbs) if err != nil { - logger.Printf("the verbs %s and %s are not answered until the assigner starts again: %v", meshwants.VerbCompositions, meshwants.VerbCompositionsDrop, err) + logger.Printf("the verbs %s, %s and %s are not answered until the assigner starts again: %v", meshwants.VerbCompositions, meshwants.VerbCompositionsDrop, meshwants.VerbDrop, err) } else { go func() { <-ctx.Done(); stopVerbs() }() } @@ -700,10 +717,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } p.ends.left.Store(&left) } - if err := serveCompositionSteps(ctx, nc, steps); err != nil { - logger.Printf("the composition step is not heard, so no composition step runs until the assigner starts again: %v", err) - } else { - logger.Printf("the composition step is heard on %s", meshwants.AcceptSubjectStep) + if steps.taken == nil { + logger.Printf("the state %s is not bound, so no composition step is pulled until the assigner starts again", process.TakenState) + } else if err := serveStepQueue(ctx, nc, js, steps, func(d time.Duration) { ackWait.Store(int64(d)) }); err != nil { + logger.Printf("the composition step is not pulled, so no composition step runs until the assigner starts again: %v", err) } } if bus != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/stepqueue.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/stepqueue.go new file mode 100644 index 000000000..0b9fdf309 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/stepqueue.go @@ -0,0 +1,318 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// **The seat mesh-wants takes its asks from a work queue and says ask-taken** (design 47 H2 §777, §775, §778; ADR 0525 §2, +// §6; ADR 0668 §2; H-A-381, H-A-401, H-A-404): the broker derives the seat's stream and its one shared worker from the seat's +// definition; the assigner pulls every ask waiting at once, takes each once by its id in its state taken (key +// ., for ask-taken-bound), says ask-taken at the pull, sends in-progress while the step runs and acknowledges an +// ask once it said the ask's end. An ask put while the assigner is down waits on the stream. The seat's act drop ends an ask +// its caller no longer waits for: one running here is cancelled, one not pulled yet is kept in taken as dropped. for +// ask-taken-bound, and either ends its step could not run naming the drop and who asked it. + +// askTakenBoundSetting is the setting the taken key's time to live is (H-A-381 (2)): the holder never receives the +// caller's bound, so the largest caller's, the runner's ask-unanswered-bound, is its default. +const ( + askTakenBoundSetting = "ask-taken-bound" + defaultAskTakenBound = 10 * time.Minute +) + +// droppedPrefix keys a drop of an ask not pulled yet in the state taken (H-A-404). +const droppedPrefix = "dropped." + +// takenStore is the state taken (process.State): an ask taken once by its id, and the drops of asks not pulled yet. +type takenStore interface { + TakeOnce(seat, ask string, bound time.Duration) (bool, error) + Get(key string) (*stdio.StateEntry, error) + RenewTTL(key string, value any, ttl time.Duration) (uint64, error) + Delete(key string) error +} + +// queueMsg is what the steps read of a message pulled from the seat's work queue (jetstream.Msg). +type queueMsg interface { + Data() []byte + Headers() nats.Header + Ack() error + Nak() error + Term() error + InProgress() error +} + +// askDrop is one drop of an ask: who asked it, why and when; the cause of a running step's end. +type askDrop struct { + By string `json:"by,omitempty"` + Why string `json:"why"` + At time.Time `json:"at"` +} + +func (d *askDrop) Error() string { + by := d.By + if by == "" { + by = "its asker" + } + return fmt.Sprintf("the ask was dropped by %s: %s", by, d.Why) +} + +// askIDOf is the ask's id a put carries: its event id, else its message id. +func askIDOf(h nats.Header) string { + if id := h.Get("x-event-id"); id != "" { + return id + } + return h.Get(nats.MsgIdHdr) +} + +// pulled handles one ask pulled from the seat's work queue: an ask that is no composition or validate step with an id is +// terminated, said; one dropped before it was pulled ends could not run naming the drop; one running here already (handed +// again while it runs) is held for that run, which acknowledges it at its end; any other is taken once by its id (ask-taken +// said the first time) and run, in progress while it runs. A repeat of an id taken before that runs here no longer is run +// again against the runs as they stand (H-A-381 (1), H-A-401): its composition runs are joined, never run twice. +func (c *compositionSteps) pulled(ctx context.Context, m queueMsg) { + id := askIDOf(m.Headers()) + var run meshwants.StepRun + err := json.Unmarshal(m.Data(), &run) + how := c.howOf(run.Step, false) + if err != nil || id == "" || how == nil { + c.logf("a step ask on the seat %s's work queue that is no composition or validate step with an id is ended: %v", meshwants.Seat, err) + _ = m.Term() + return + } + if d := c.droppedBefore(id); d != nil { + c.hold(id, m) + c.logf("the ask %s (%s) was dropped before it was pulled: %v", id, run.Pipeline, d) + c.sayEnd(id, run, nil, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: d.Error()}, "") + if err := c.taken.Delete(droppedPrefix + id); err != nil { + c.logf("the drop of the ask %s cannot be deleted from the state %s: %v", id, process.TakenState, err) + } + return + } + if c.running(id) { + c.hold(id, m) + c.logf("the ask %s (%s) was handed again while it runs here: its run answers it", id, run.Pipeline) + return + } + first, err := c.taken.TakeOnce(meshwants.Seat, id, c.takenBound) + if err != nil { + c.logf("the ask %s (%s) is not taken, the state %s refused it: %v; it is handed again", id, run.Pipeline, process.TakenState, err) + _ = m.Nak() + return + } + c.hold(id, m) + if first { + at := c.now().UTC() + if err := c.say(meshwants.EventAskTaken+"."+id, meshwants.AskTakenEvent{Ask: id, Node: c.node, At: at}, id+".taken"); err != nil { + c.logf("saying ask-taken of %s: %v", id, err) + } + } else { + c.logf("the ask %s (%s) was taken before: it is run again against its composition runs as they stand", id, run.Pipeline) + } + go c.say1(ctx, id, run, how, "") +} + +// hold keeps m for the step asked under id: acknowledged once its end is said, in progress while it runs. +func (c *compositionSteps) hold(id string, m queueMsg) { + c.mu.Lock() + defer c.mu.Unlock() + if c.held == nil { + c.held = map[string][]queueMsg{} + } + c.held[id] = append(c.held[id], m) +} + +// settle acknowledges (ack) or hands again (nak) every message held for id. +func (c *compositionSteps) settle(id string, ack bool) { + c.mu.Lock() + ms := c.held[id] + delete(c.held, id) + c.mu.Unlock() + for _, m := range ms { + if ack { + _ = m.Ack() + } else { + _ = m.Nak() + } + } +} + +// inProgressEvery says every message held for id in progress every d, until the returned stop is called. +func (c *compositionSteps) inProgressEvery(id string) (stop func()) { + every := time.Duration(0) + if c.inProgress != nil { + every = c.inProgress() + } + if every <= 0 { + return func() {} + } + ticks := c.ticks + if ticks == nil { + ticks = func(d time.Duration) (<-chan time.Time, func()) { t := time.NewTicker(d); return t.C, t.Stop } + } + tick, stopTicks := ticks(every) + done, ended := make(chan struct{}), make(chan struct{}) + go func() { + defer close(ended) + defer stopTicks() + for { + select { + case <-done: + return + case <-tick: + c.mu.Lock() + ms := append([]queueMsg(nil), c.held[id]...) + c.mu.Unlock() + for _, m := range ms { + _ = m.InProgress() + } + } + } + }() + return func() { close(done); <-ended } +} + +// droppedBefore is the drop kept for an ask not pulled yet, nil for none. +func (c *compositionSteps) droppedBefore(id string) *askDrop { + if c.taken == nil { + return nil + } + e, err := c.taken.Get(droppedPrefix + id) + if err != nil { + c.logf("whether the ask %s was dropped cannot be read from the state %s: %v; it is taken", id, process.TakenState, err) + return nil + } + if e == nil { + return nil + } + var d askDrop + if json.Unmarshal(e.Value, &d) != nil { + return nil + } + return &d +} + +// dropAsk is the seat's act drop (§778, H-A-404): a step running here for the ask is cancelled with the drop as its cause and +// ends could not run naming it; one not running here is kept in taken as dropped. for ask-taken-bound, so its pull ends +// it the same way. It answers whether the ask was held here. +func (c *compositionSteps) dropAsk(caller string, in meshwants.DropInput) (meshwants.DropAnswer, error) { + id, why := strings.TrimSpace(in.ID), strings.TrimSpace(in.Why) + if id == "" || why == "" { + return nil, errors.New("give the ask's id and why it is dropped") + } + by := strings.TrimSpace(in.By) + if by == "" { + by = caller + } + d := &askDrop{By: by, Why: why, At: c.now().UTC()} + c.mu.Lock() + stop := c.stops[id] + c.mu.Unlock() + if stop != nil { + stop(d) + c.logf("the ask %s running here is dropped: %v", id, d) + return meshwants.DropAnswer{"held": true}, nil + } + if c.taken == nil { + return nil, fmt.Errorf("the state %s is not bound, so the drop of %s cannot be kept", process.TakenState, id) + } + if _, err := c.taken.RenewTTL(droppedPrefix+id, d, c.takenBound); err != nil { + return nil, fmt.Errorf("writing the drop of %s in the state %s: %w", id, process.TakenState, err) + } + c.logf("the ask %s, not running here, is kept dropped until it is pulled: %v", id, d) + return meshwants.DropAnswer{"held": false}, nil +} + +// dropVerb is the process's handler of the seat mesh-wants' act drop. +func dropVerb(c *compositionSteps) map[string]process.SeatHandler { + return map[string]process.SeatHandler{ + meshwants.VerbDrop: func(caller string, raw json.RawMessage) (any, error) { + var in meshwants.DropInput + if err := json.Unmarshal(raw, &in); err != nil { + return nil, fmt.Errorf("the arguments are not JSON: %w", err) + } + return c.dropAsk(caller, in) + }, + } +} + +// inProgressOf is the seat's in-progress as its definition names it, once the seat list is read; before, a third of the +// worker's ack wait, which the broker derived from the same definition (60 s and 20 s, §777; H-A-401). +func inProgressOf(def func() (seatdef.Definition, bool), ackWait func() time.Duration) func() time.Duration { + return func() time.Duration { + if d, ok := def(); ok { + if _, worker, ok, err := buscompose.SeatQueue(meshwants.Seat, d.Accepts); err == nil && ok && worker.InProgressSeconds > 0 { + return time.Duration(worker.InProgressSeconds) * time.Second + } + } + return ackWait() / 3 + } +} + +// serveStepQueue pulls the seat's work queue through its one shared worker, every ask waiting at once (at most the worker's +// max ack pending), once the broker made it: a worker not there yet is said, and bound when the bus says it was created. +func serveStepQueue(ctx context.Context, conn process.Subscriber, js jetstream.JetStream, c *compositionSteps, ackWait func(time.Duration)) error { + stream, worker := buscompose.SeatQueueStreamName(meshwants.Seat), buscompose.SeatQueueWorkerName(meshwants.Seat) + var once sync.Once + bind := func() bool { + cons, err := js.Consumer(ctx, stream, worker) + if err != nil { + return false + } + bound := false + once.Do(func() { + bound = true + info := cons.CachedInfo() + if info != nil && info.Config.AckWait > 0 { + ackWait(info.Config.AckWait) + } + pending := 1000 + if info != nil && info.Config.MaxAckPending > 0 { + pending = info.Config.MaxAckPending + } + cc, err := cons.Consume(func(m jetstream.Msg) { c.pulled(ctx, m) }, jetstream.PullMaxMessages(pending)) + if err != nil { + c.logf("the seat %s's work queue %s is not pulled: %v", meshwants.Seat, worker, err) + return + } + c.logf("the composition step is pulled from %s on %s", worker, stream) + go func() { <-ctx.Done(); cc.Stop() }() + }) + return bound + } + if bind() { + return nil + } + c.logf("the seat %s's work queue %s on %s is not there yet: its asks wait on it, and it is pulled when the bus says it was created", meshwants.Seat, worker, stream) + sub, err := conn.Subscribe("$JS.EVENT.ADVISORY.CONSUMER.CREATED."+stream+"."+worker, func(*nats.Msg) { go bind() }) + if err != nil { + return fmt.Errorf("hearing the creation of the seat %s's work queue: %w", meshwants.Seat, err) + } + go func() { <-ctx.Done(); _ = sub.Unsubscribe() }() + if bind() { // made between the first look and the subscription + _ = sub.Unsubscribe() + } + return nil +} + +// askTakenBound reads the setting ask-taken-bound (§781). +func askTakenBound() (time.Duration, error) { + d, err := stdio.SettingDuration(askTakenBoundSetting, defaultAskTakenBound) + if err != nil { + return 0, errors.New("the assigner's " + err.Error()) + } + return d, nil +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go index c71e7dbd5..94437bcb3 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go @@ -22,6 +22,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/seatlist" "git.novox.be/novox/mesh-sdk/go/stateobjects" "mesh-delivery-assigner/internal/buckets" @@ -416,6 +417,7 @@ func serveTools() error { tools = append(tools, whyTools(runtimeStore{})...) tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...) tools = append(tools, wantDropTools(runtimeStore{}, stdio.Ask)...) + tools = append(tools, asksTools(stdio.QueueWaiting, stdio.State(process.TakenState), stdio.Ask)...) // the seat's work queue (§777) return stdio.Serve("", tools) } diff --git a/modules/mesh-delivery-assigner/internal/buckets/buckets.go b/modules/mesh-delivery-assigner/internal/buckets/buckets.go index a894baafb..a3aacb739 100644 --- a/modules/mesh-delivery-assigner/internal/buckets/buckets.go +++ b/modules/mesh-delivery-assigner/internal/buckets/buckets.go @@ -44,6 +44,10 @@ func Inputs() string { return own(InputsState) } func CompositionRuns() string { return own(CompositionRunsState) } func Policies() string { return own(PoliciesState) } +// Taken is the bucket of the state taken (design 47 H2 §775, §777): the asks of the seat mesh-wants taken once, and their +// drops; named as mesh-sdk's process.TakenState. +func Taken() string { return own("taken") } + // AsksHistory is the revisions the bucket of asks keeps per key. const AsksHistory = 5 diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 8a3db0c22..b7bc88695 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -70,7 +70,7 @@ func TestManifest(t *testing.T) { } if want := map[string]int{"asks": 5, "differences": 64, "puts": 20, "wants": 20, "assignments": 64, "placement": 64, "copies": 1, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5, views.MarksState: 1, views.StatusState: 1, - "engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "inputs": 1, "composition-runs": 1}; !reflect.DeepEqual(states, want) { + "engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "inputs": 1, "composition-runs": 1, "taken": 1}; !reflect.DeepEqual(states, want) { t.Errorf("state = %v, want %v", states, want) } if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 { @@ -93,7 +93,7 @@ func TestManifest(t *testing.T) { // revision, entry and difference from its revisions kept as objects in the old section, form 1, capped at 512 MiB. if len(m.Claims) != 2 || m.Claims[0].Name != meshmoduleassigner.Seat || m.Claims[0].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[0].Serves, []string{"assignments", "proposals", "propose", "withdraw", "again", "placement"}) || - m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input", "compositions", "compositions-drop"}) { + m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input", "compositions", "compositions-drop", "drop"}) { t.Errorf("the assigner claims %+v, want mesh-module-assigner with its five verbs and mesh-wants with its five", m.Claims) } if len(m.Objects) != 1 || m.Objects[0].Name != "assignment-revisions" || m.Objects[0].MaxMiB != 512 || m.Objects[0].Form != 1 { @@ -105,7 +105,7 @@ func TestManifest(t *testing.T) { // which lists and asks again what each node's engine reads. if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_waits", "assigner_inputs", "assigner_inputs_drop", "assigner_compositions", "assigner_compositions_drop", "assignments", "seat-holders", "wants", "asks", "assigner_read_assignments", "assigner_read_seat_holders", "assigner_read_wants", "assigner_read_findings", "assigner_read_differences", "assigner_read_copies", - "assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_want_drop"}) { + "assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_want_drop", "assigner_asks", "assigner_ask_drop"}) { t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads, the seat list's two (ADR 0636 §7) assigner_ask_again (ADR 0668 §5) and assigner_why (design 47 G A-439)", m.Tools) } if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs { @@ -172,9 +172,11 @@ func TestManifest(t *testing.T) { } file, _ := os.ReadFile("module.json") if err := json.Unmarshal(file, &raw); err != nil || raw.Config["outside-wait-minutes"].Default != float64(30) || raw.Config["outside-settle-seconds"].Default != float64(90) || - raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 10 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) || + raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 13 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) || raw.Config["composition-runs-cap-mib"].Default != float64(16) || raw.Config["composition-runs-kept"].Default != "24h" || - raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) { + raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) || + // design 47 H2 §775, H-A-381 (2): the taken key's bound and the kept-call pace, runtime settings + raw.Config["ask-taken-bound"].Default != "10m" || raw.Config["kept-call-pace-first"].Default != "1s" || raw.Config["kept-call-pace-max"].Default != "30s" { t.Errorf("config = %+v (%v), want outside-wait-minutes of 30, outside-settle-seconds of 90 and ask-operator-for of take-off,remove-node,stops (A-425a)", raw.Config, err) } for _, f := range []struct{ env, path, key string }{ diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 329791381..e1502c7d3 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -27,7 +27,8 @@ "difference", "input", "compositions", - "compositions-drop" + "compositions-drop", + "drop" ] } ], @@ -129,7 +130,9 @@ "assigner_ask_again", "assigner_written", "assigner_why", - "assigner_want_drop" + "assigner_want_drop", + "assigner_asks", + "assigner_ask_drop" ], "state": [ { "name": "assignments", "history": 64 }, @@ -152,7 +155,8 @@ { "name": "waits", "history": 5 }, { "name": "put-waits", "history": 1 }, { "name": "inputs", "history": 1, "max-mib": 64 }, - { "name": "composition-runs", "history": 1, "key-ttl": true, "max-mib": 64 } + { "name": "composition-runs", "history": 1, "key-ttl": true, "max-mib": 64 }, + { "name": "taken", "history": 1, "key-ttl": true, "max-mib": 8 } ], "config": { "outside-wait-minutes": { @@ -226,6 +230,27 @@ "maximum": 1000, "trusted": false, "default": 20 + }, + "ask-taken-bound": { + "type": "string", + "description": "how long the assigner keeps the id of an ask it took from the seat mesh-wants' work queue, so a repeat of it is taken once, and the drop of an ask not pulled yet, a Go duration such as 10m (design 47 H2 §775, §778; H-A-381 (2), H-A-404).", + "trusted": false, + "default": "10m", + "runtime": true + }, + "kept-call-pace-first": { + "type": "string", + "description": "the first wait before a kept call met by a transport fault is made again, doubled per call up to kept-call-pace-max, a Go duration (design 47 H2 §775).", + "trusted": false, + "default": "1s", + "runtime": true + }, + "kept-call-pace-max": { + "type": "string", + "description": "the most a kept call's wait between calls grows to, a Go duration (design 47 H2 §775).", + "trusted": false, + "default": "30s", + "runtime": true } }, "objects": [ From e216e94029d18522219e5adb93612eb501f3fe33 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:07:22 +0100 Subject: [PATCH 02/12] A1: the assigner asks each machine's compose-candidate as a kept call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A composition step ended at a node-engine's first "no responders": the engine between two lives gave no input. The holder of mesh-wants now asks compose-candidate through mesh-sdk's kept chunked call, under the ask id . the engine takes once, called again at the kept-call pace through transport faults until compose-ask-bound (a runtime setting, 10s). Builds: design 47 H2 §775; H-A-405, H-A-364 --- .../compositioninputs.go | 57 ++++++++++++------- .../compositioninputs_test.go | 14 +++-- .../mesh-delivery-assigner/compositionrun.go | 8 +-- .../compositionrun_test.go | 3 +- .../cmd/mesh-delivery-assigner/main.go | 6 +- .../cmd/mesh-delivery-assigner/pass_test.go | 4 ++ .../cmd/mesh-delivery-assigner/puts.go | 3 + .../mesh-delivery-assigner/manifest_test.go | 4 +- modules/mesh-delivery-assigner/module.json | 7 +++ 9 files changed, 75 insertions(+), 31 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go index 52c522630..afb8b1fb2 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs.go @@ -15,6 +15,8 @@ import ( nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" ) // The inputs of a composition run (design 47 G §354, §718, A-243, A-252): one request per candidate and machine, carrying that @@ -72,28 +74,44 @@ type nodeInput struct { // compositionInputs asks the machines of a step for their inputs. type compositionInputs struct { machines func(module, branch string) []string - // callFile asks a seat verb marked chunks, addressed to one node as `seat:.@`, and answers the verb's answer with - // the whole file received added as its member `file`, base64 (process.Conn.CallFile; design 47 G A-237b, A-237c, A-249). The call - // token is the body's `call`. - callFile func(key string, body any) (json.RawMessage, error) - bound time.Duration + // callFile asks a seat verb marked chunks as a kept call (process.Conn.CallFileKept; design 47 H2 §775, H-A-405), addressed to + // one node as `seat:.@`, carrying the ask's id and stopping at its bound, and answers the verb's answer with + // the whole file received added as its member `file`, base64 (design 47 G A-237b, A-237c, A-249). The call token is the + // body's `call`. + callFile func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) + // bound is compose-ask-bound: how long a machine's compose-candidate is asked again through a transport fault (§775). + bound time.Duration } -// of asks the machine node for its input with the candidate's key (§718): asked only by the composition step that starts the -// machine's composition run, once per candidate and machine. A node that is silent, unready or answers no input has Why set. -func (c *compositionInputs) of(ctx context.Context, node, candidate string) nodeInput { +// composeAskBoundSetting is the holder's bound on its compose-candidate asks (design 47 H2 §775): past it the machine gave no +// input, naming every call. +const ( + composeAskBoundSetting = "compose-ask-bound" + defaultComposeAskBound = 10 * time.Second +) + +// composeAskBound reads the setting compose-ask-bound (§781). +func composeAskBound() (time.Duration, error) { + d, err := stdio.SettingDuration(composeAskBoundSetting, defaultComposeAskBound) + if err != nil { + return 0, errors.New("the assigner's " + err.Error()) + } + return d, nil +} + +// composeAskID is the id of the compose-candidate ask of the step asked under stepAsk to node (H-A-405 (2)): one per node per +// step, so the engines' mesh-wide state taken keeps each apart, and a repeat to the same node carries the same id. +func composeAskID(stepAsk, node string) string { return stepAsk + "." + node } + +// of asks the machine node for its input with the candidate's key (§718), as a kept call under the id of stepAsk, the step +// that starts the machine's composition run: asked once per candidate and machine, again only through a transport fault until +// compose-ask-bound (§775). A node that is silent past the bound, unready or answers no input has Why set. +func (c *compositionInputs) of(ctx context.Context, node, candidate, stepAsk string) nodeInput { bound := c.bound if bound <= 0 { - bound = nec.AnswerBound + bound = defaultComposeAskBound } - return c.one(ctx, bound, node, candidate) -} - -// one asks one node for its input. -func (c *compositionInputs) one(ctx context.Context, bound time.Duration, node, key string) nodeInput { in := nodeInput{Node: node} - at, cancel := context.WithTimeout(ctx, bound) - defer cancel() type reply struct { raw json.RawMessage err error @@ -106,13 +124,14 @@ func (c *compositionInputs) one(ctx context.Context, bound time.Duration, node, got <- reply{nil, err} return } - raw, err := c.callFile(composeCandidateKey+"@"+node, ne.ComposeCandidateInput{Candidate: key, Input: true, Call: hex.EncodeToString(bits[:])}) + raw, err := c.callFile(process.KeptAsk{ID: composeAskID(stepAsk, node), Bound: bound}, composeCandidateKey+"@"+node, + ne.ComposeCandidateInput{Candidate: candidate, Input: true, Call: hex.EncodeToString(bits[:])}) got <- reply{raw, err} }() var raw json.RawMessage select { - case <-at.Done(): - in.Why = "no answer within its bound: " + at.Err().Error() + case <-ctx.Done(): + in.Why = "its step's wait ended: " + context.Cause(ctx).Error() return in case r := <-got: if r.err != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go index d403de527..596bcbab5 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositioninputs_test.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "errors" + "git.novox.be/novox/mesh-sdk/go/process" "regexp" "strings" "sync" @@ -71,11 +72,12 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) { // the machines are asked side by side, so the record of what each was asked is guarded. var mu sync.Mutex asked := map[string]string{} - call := func(key string, body any) (json.RawMessage, error) { + asks := map[string]process.KeptAsk{} + call := func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) { raw, _ := json.Marshal(body) key, node, _ := strings.Cut(key, "@") // addressed to one node (A-237b) mu.Lock() - asked[node] = key + " " + string(raw) + asked[node], asks[node] = key+" "+string(raw), ask mu.Unlock() switch node { case "g14": @@ -98,6 +100,10 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) { if got[0].Node != "ace" || got[0].Why != "" || !strings.Contains(string(got[0].Body), "12345678901234567890") { t.Errorf("ace: %+v", got[0]) } + // Design 47 H2 §775, H-A-405: each machine is asked as a kept call under the step's ask with the node, until compose-ask-bound. + if asks["ace"] != (process.KeptAsk{ID: "step-1.ace", Bound: time.Second}) || asks["g14"].ID != "step-1.g14" { + t.Errorf("the kept asks %+v", asks) + } if !strings.Contains(got[1].Why, "no responders") || !strings.Contains(got[2].Why, "unready") || got[1].Body != nil || got[2].Body != nil { t.Errorf("g14 and novox: %+v %+v", got[1], got[2]) } @@ -115,7 +121,7 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) { func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) { input := `{"n":12345678901234567890,"big":"` + strings.Repeat("x", 1<<20) + `"}` file := base64.StdEncoding.EncodeToString([]byte(input)) - call := func(key string, body any) (json.RawMessage, error) { + call := func(_ process.KeptAsk, key string, body any) (json.RawMessage, error) { _, node, _ := strings.Cut(key, "@") answer := `{"candidate":"run-r1","node":"` + node + `","verdict":"composed","modules":[],"own":{},"took":1,"v":1,"chunks":3` if node == "g14" { @@ -155,7 +161,7 @@ func gatherAll(c *compositionInputs, run meshwants.StepRun, away func(string) bo continue } wg.Add(1) - go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key) }() + go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key, "step-1") }() } wg.Wait() return out, nil diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go index f5d3088be..daddb921d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go @@ -126,7 +126,7 @@ func (r *compositionRun) run(ctx context.Context, id string, step meshwants.Step continue } if j.started { - go r.start(ctx, j.key, j.run, prep) + go r.start(ctx, j.key, j.run, prep, id) } keys = append(keys, j.key) } @@ -241,9 +241,9 @@ func (p *runPrep) get(ctx context.Context) error { return p.err } -// start composes the machine of the run it started (§718): the node's input of this moment, its request kept once in `inputs`, +// start composes the machine of the run the step asked under stepAsk started (§718): the node's input of this moment, its request kept once in `inputs`, // the composer program chosen and recorded with the request's digest, and one run ask put. A fault ends the run could not run. -func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep) { +func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep, stepAsk string) { node, ask := run.Machine, run.Ask end := func(said string) { if _, err := r.runs.endRun(key, ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil); err != nil { @@ -269,7 +269,7 @@ func (r *compositionRun) start(ctx context.Context, key string, run meshwants.Co end(node + ": " + err.Error()) return } - in := r.inputs.of(ctx, node, run.Candidate) + in := r.inputs.of(ctx, node, run.Candidate, stepAsk) // a kept call under the starting step's ask with the node (H-A-405) if in.Why != "" { end(node + " gave no composition input: " + in.Why) return diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go index 60cc21601..2d94118fc 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "git.novox.be/novox/mesh-sdk/go/process" "maps" "slices" "strings" @@ -103,7 +104,7 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) * } return nil, errors.New("no such verb " + key) } - engines := func(key string, _ any) (json.RawMessage, error) { + engines := func(_ process.KeptAsk, key string, _ any) (json.RawMessage, error) { _, node, _ := strings.Cut(key, "@") // addressed to one node (A-237b) return json.RawMessage(`{"candidate":"main.7","node":"` + node + `","verdict":"composed","input":{"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 548b74c34..cc6e8168b 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -671,7 +671,11 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } // The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine. if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil { - run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, + askBound, err := composeAskBound() + if err != nil { + return err + } + run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFileKept, bound: askBound}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, trunk: trunkOf(iv, w.trunkName), newest: iv.newestOf(), watched: iv.watchedOf, assigned: w.assignedOn, assignment: w.assignmentOf, findings: findings, running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf} // The seat's work queue (design 47 H2 §777): pulled through its one worker, each ask taken once in the state taken. takenBound, err := askTakenBound() diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index cfed17a95..3a82d04a2 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-assigner/internal/buckets" "mesh-delivery-assigner/internal/puts" "os" @@ -295,6 +296,9 @@ func (b *fakeBus) CallOn(node, key string, body any) (json.RawMessage, error) { } func (b *fakeBus) CallKept(key string, body any) (json.RawMessage, error) { return b.Call(key, body) } func (b *fakeBus) CallFile(key string, body any) (json.RawMessage, error) { return b.Call(key, body) } +func (b *fakeBus) CallFileKept(_ process.KeptAsk, key string, body any) (json.RawMessage, error) { + return b.Call(key, body) +} func (b *fakeBus) Emit(eventType string, body any) error { b.events = append(b.events, fmt.Sprintf("%s %+v", eventType, body)) return nil diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go index b42fa8594..a3deaf66d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go @@ -21,6 +21,7 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-assigner/internal/assign" ) @@ -44,6 +45,8 @@ type caller interface { CallKept(key string, body any) (json.RawMessage, error) // CallFile is Call of a verb marked chunks, its answer read whole (process.Conn.CallFile; design 47 G A-291). CallFile(key string, body any) (json.RawMessage, error) + // CallFileKept is CallFile as a kept call carrying its ask's id, until its bound (process.Conn.CallFileKept; H2 §775). + CallFileKept(ask process.KeptAsk, key string, body any) (json.RawMessage, error) Emit(eventType string, body any) error } diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index b7bc88695..1bdb3c35d 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -172,11 +172,11 @@ func TestManifest(t *testing.T) { } file, _ := os.ReadFile("module.json") if err := json.Unmarshal(file, &raw); err != nil || raw.Config["outside-wait-minutes"].Default != float64(30) || raw.Config["outside-settle-seconds"].Default != float64(90) || - raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 13 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) || + raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 14 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) || raw.Config["composition-runs-cap-mib"].Default != float64(16) || raw.Config["composition-runs-kept"].Default != "24h" || raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) || // design 47 H2 §775, H-A-381 (2): the taken key's bound and the kept-call pace, runtime settings - raw.Config["ask-taken-bound"].Default != "10m" || raw.Config["kept-call-pace-first"].Default != "1s" || raw.Config["kept-call-pace-max"].Default != "30s" { + raw.Config["ask-taken-bound"].Default != "10m" || raw.Config["kept-call-pace-first"].Default != "1s" || raw.Config["kept-call-pace-max"].Default != "30s" || raw.Config["compose-ask-bound"].Default != "10s" { t.Errorf("config = %+v (%v), want outside-wait-minutes of 30, outside-settle-seconds of 90 and ask-operator-for of take-off,remove-node,stops (A-425a)", raw.Config, err) } for _, f := range []struct{ env, path, key string }{ diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index e1502c7d3..699f5dfd1 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -238,6 +238,13 @@ "default": "10m", "runtime": true }, + "compose-ask-bound": { + "type": "string", + "description": "how long the assigner asks a machine's node-engine compose-candidate again through a transport fault before the machine gave no input, a Go duration such as 10s (design 47 H2 §775).", + "trusted": false, + "default": "10s", + "runtime": true + }, "kept-call-pace-first": { "type": "string", "description": "the first wait before a kept call met by a transport fault is made again, doubled per call up to kept-call-pace-max, a Go duration (design 47 H2 §775).", From 48aa36120db2eb012cb16c9e9abb8b7278282ebe Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:07:50 +0100 Subject: [PATCH 03/12] A3: an answered validate or composition ask is one journal line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The journal said a step's end only when it failed to say it, so an ask's answer could not be followed by reading the journal. The assigner writes "answered " where the step's end is said, the one place a validate or composition ask is answered. Builds: design 47 H2 §792 --- .../mesh-delivery-assigner/compositionstep.go | 2 ++ .../compositionstep_test.go | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go index 04615205f..6bf171b15 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep.go @@ -183,6 +183,8 @@ func (c *compositionSteps) sayEnd(id string, run meshwants.StepRun, started *tim return } c.settle(id, true) // the ask's end is said: its pulled messages leave the seat's work queue (§777) + // One journal line per ask answered, in the one place it is answered (design 47 H2 §792). + c.logf("answered %s %s %s %s %s", id, run.Pipeline, run.Module, step, o.Outcome) if c.ended != nil { c.ended(id, run, ended) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go index edf0d478f..d38980356 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "slices" "strings" "sync" @@ -137,7 +138,14 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) { said := newSaidEvents() at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC) var asked meshwants.StepRun - c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf, + var lines []string + var linesMu sync.Mutex + logf := func(f string, a ...any) { + linesMu.Lock() + lines = append(lines, fmt.Sprintf(f, a...)) + linesMu.Unlock() + } + c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: logf, run: func(_ context.Context, id string, run meshwants.StepRun) stepOutcome { asked = run return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"}, @@ -175,6 +183,13 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) { len(end.Shared) != 1 || end.Shared[0] != (meshwants.Shared{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}) { // §723, §729 t.Fatalf("step-ended %+v", end) } + // Design 47 H2 §792: the ask answered is one journal line, said where its end is said. + linesMu.Lock() + answered := slices.DeleteFunc(slices.Clone(lines), func(l string) bool { return !strings.HasPrefix(l, "answered ") }) + linesMu.Unlock() + if !slices.Equal(answered, []string{"answered ask-1 main.4.m m composition failed"}) { + t.Errorf("the journal says %q, want one answered line", answered) + } // Design 47 H2 §777, §775 (H-A-381, H-A-401, H-A-404): the asks of the seat's work queue, taken once by id, said ask-taken at // the pull, in progress while they run, acknowledged once their end is said, and ended by the seat's drop. t.Run("work queue", queueCases) From 7c9da7d0cb5cbfd84ebd039385ebe003178c69b7 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:14:05 +0100 Subject: [PATCH 04/12] A5: a failed keeping of the assignment graph runs again, and by hand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A keeping that failed was handed again only by the next pass, so with no change of inputs no revision was kept and no want said as wanted. A failed keeping now runs again after keeping-retry (a runtime setting, 30s, the holder's own deadline), unless a newer revision was handed meanwhile, and raises keeping-failed (scope module) at the third failure in a row, cleared at the next keeping that succeeds. The seat mesh-wants gains the act keep-now, which the assigner's process serves: it keeps the pass's newest computed revision at once, one keeping at a time, and answers the revision kept or the fault met; the tool assigner_keep_now writes the hand act first and calls it. Builds: design 47 H2 §813 (2); H-A-417 --- .../cmd/mesh-delivery-assigner/asks_tools.go | 31 ++++ .../cmd/mesh-delivery-assigner/graphkept.go | 162 +++++++++++++++++- .../mesh-delivery-assigner/graphkept_test.go | 64 +++++++ .../cmd/mesh-delivery-assigner/main.go | 9 +- .../cmd/mesh-delivery-assigner/tools.go | 1 + .../mesh-delivery-assigner/manifest_test.go | 8 +- modules/mesh-delivery-assigner/module.json | 11 +- .../go/seats/meshwants/contract.go | 30 +++- .../seats/mesh-wants/seat.json | 30 ++++ .../ts/seats/meshwants/index.d.ts | 17 ++ .../ts/seats/meshwants/index.js | 1 + 11 files changed, 347 insertions(+), 17 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks_tools.go index 550bfb4fb..251610e5e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks_tools.go @@ -129,3 +129,34 @@ func asksTools(waiting func(string) (stdio.Waiting, error), taken takenReader, a }}, } } + +// keepNowTool is the tool assigner_keep_now (design 47 H2 §813 (2); H-A-417): written in the hand-act log first, then the seat +// mesh-wants' act keep-now runs the keeping of the assignment graph's revisions at once in the assigner's process. +const keepNowTool = "assigner_keep_now" + +func keepNowTools(ask func(string, any) (json.RawMessage, error)) []stdio.Tool { + return []stdio.Tool{{Name: keepNowTool, Description: "Keep the assignment graph's revisions now (design 47 H2 §813 (2)): written in the " + + "hand-act log first, then the assigner keeps the pass's newest computed revision at once and answers the revision kept, or " + + "the fault the keeping met (a failed keeping runs again after keeping-retry; keeping-failed is raised at the third in a row).", + Input: schema(map[string]string{"why": "why the keeping runs now"}, []string{"why"}), + Run: func(args map[string]any) (any, error) { + why := strings.TrimSpace(arg(args, "why")) + if why == "" { + return nil, errors.New("say why the keeping runs now") + } + taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: keepNowTool, Why: why + " (" + stdio.CallerOf(args) + ")", Cause: "keep-now"}) + if err != nil { + return nil, fmt.Errorf("the hand act was not recorded, so nothing was kept: %w", err) + } + raw, err := ask("seat:"+meshwants.Seat+"."+meshwants.VerbKeepNow, meshwants.KeepNowInput{Why: why}) + if err != nil { + return nil, err + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("%s answered what is not JSON: %w", meshwants.VerbKeepNow, err) + } + out["hand-act"] = taken.Log + return out, nil + }}} +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go index b60771eb2..1eb4cc98e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go @@ -5,18 +5,25 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "slices" "strings" "sync" "time" - mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + mcatc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-assigner/internal/assign" "mesh-delivery-assigner/internal/assigngraph" + "mesh-delivery-assigner/internal/assignment" ) // graphKeeper is the pass's assignment graph (design 47 G §292, A-67): per catalog revision taken in it computes the revision @@ -28,7 +35,7 @@ type graphKeeper struct { // reads is the same store through a handle of the pass's own, for the pass's read of its fill (graphfill.go); nil reads objects. reads assigngraph.Objects // halves is every followed branch's catalog graph revision held now. - halves func() map[string]mcc.Half + halves func() map[string]mcatc.Half // mu guards what the keeping writes and the pass reads: the revision kept, full, and the need handed over (design 47 H-A-259). mu sync.Mutex // handedRev is the revision last handed to the keeping, so a pass whose inputs did not change hands nothing; handed is false @@ -46,14 +53,40 @@ type graphKeeper struct { full, started bool // revisions removes a revision that ceased to be current (revisions.go, H-A-86); nil keeps every revision. revisions *revisionRemover - // computed is the revision computed by the last pass, with its entries: its edges. - computed assigngraph.Computed + // computed is the revision computed by the last pass, with its entries: its edges; hasComputed says a pass computed one. + computed assigngraph.Computed + hasComputed bool + // keepMu runs one keeping at a time, the pass's (keepOne) or one by hand (keepNow). + keepMu sync.Mutex // The keeping off the pass loop (H-A-259): need is what waits to be kept, keeping says one runs; background runs it (nil: at // once, in the caller), and wake makes a pass once a keeping changed what the pass reads. need *keepNeed keeping bool background func(func()) wake func() + // A keeping that fails runs again after retry, keeping-retry, by after (time.AfterFunc; a test hands its own); failures + // counts the failures in a row, and keeping-failed is raised at the third (design 47 H2 §813 (2)). + retry time.Duration + after func(d time.Duration, f func()) + failures int +} + +// The keeping's setting and finding (design 47 H2 §813 (2); ADR 0682 §11: the holder's own deadline). +const ( + keepingRetrySetting = "keeping-retry" + defaultKeepingRetry = 30 * time.Second + kindKeepingFailed = "keeping-failed" + // keepingFailedAt is the failures in a row that raise keeping-failed. + keepingFailedAt = 3 +) + +// keepingRetry reads the setting keeping-retry (§781). +func keepingRetry() (time.Duration, error) { + d, err := stdio.SettingDuration(keepingRetrySetting, defaultKeepingRetry) + if err != nil { + return 0, errors.New("the assigner's " + err.Error()) + } + return d, nil } // keepNeed is one need of the keeping: a computed revision to keep (nil: none), and whether the revisions not current are judged. @@ -99,7 +132,9 @@ func (p *passer) keepGraph(ctx context.Context) string { return "" } c := assigngraph.Compute(p.graphInputs()) - g.computed = c + g.mu.Lock() + g.computed, g.hasComputed = c, true + g.mu.Unlock() for _, u := range c.Uncomputed { if g.said != u { g.said = u @@ -177,6 +212,8 @@ func (p *passer) needKeeping(ctx context.Context, n keepNeed) { // revisions that ceased to be current; it says its duration in the journal, and wakes a pass when it changed what the pass reads. func (p *passer) keepOne(ctx context.Context, n keepNeed) { g := p.graph + g.keepMu.Lock() + defer g.keepMu.Unlock() start := time.Now() changed := false defer func() { @@ -199,6 +236,12 @@ func (p *passer) keepOne(ctx context.Context, n keepNeed) { // keepRevision keeps c when the store holds it, and says whether a revision was kept now. func (p *passer) keepRevision(ctx context.Context, c assigngraph.Computed) bool { + fresh, _ := p.keepRevisionOf(ctx, c) + return fresh +} + +// keepRevisionOf is keepRevision answering the fault the keeping met too. +func (p *passer) keepRevisionOf(ctx context.Context, c assigngraph.Computed) (bool, error) { g := p.graph again := func() { // the next pass hands this revision again g.mu.Lock() @@ -208,10 +251,11 @@ func (p *passer) keepRevision(ctx context.Context, c assigngraph.Computed) bool used, _, _ := g.fillOf(ctx) rev, fresh, err := assigngraph.Keep(ctx, g.objects, c, meshwants.EventAssignmentRevision, p.clock()) if err != nil { - p.logf("the assignment graph revision is not kept, so no want is said as wanted until it is: %v", err) again() - return false + p.keepingFailed(ctx, c, err) + return false, err } + p.keepingSucceeded() var source int64 for _, cat := range rev.Catalogs { if cat.Branch == p.w.trunkName() { @@ -222,14 +266,54 @@ func (p *passer) keepRevision(ctx context.Context, c assigngraph.Computed) bool g.kept, g.ok, g.source = rev, true, source g.mu.Unlock() if !fresh { - return false + return false, nil } p.fillingAfter(ctx, used) id := fmt.Sprintf("%s-%s-%d", meshwants.EventAssignmentRevision, rev.Kind, rev.Revision) if err := p.wantsSay(meshwants.EventAssignmentRevision, assigngraph.Event(rev), id); err != nil { p.logf("the assignment revision %d is kept but not said: %v", rev.Revision, err) } - return true + return true, nil +} + +// keepNow is the seat mesh-wants' act keep-now (design 47 H2 §813 (2); H-A-417): the keeping of the pass's newest computed +// revision, run at once, one at a time with the pass's; it answers the revision kept, or the fault the keeping met (a failed +// one is run again after keeping-retry, as every keeping is). The hand act is the tool's to write, before it calls. +func (p *passer) keepNow(ctx context.Context, in meshwants.KeepNowInput) (meshwants.KeepNowAnswer, error) { + g := p.graph + if strings.TrimSpace(in.Why) == "" { + return meshwants.KeepNowAnswer{}, errors.New("say why the keeping runs now") + } + if g == nil || g.objects == nil { + return meshwants.KeepNowAnswer{}, errors.New("the assigner keeps no assignment graph here") + } + g.mu.Lock() + c, ok := g.computed, g.hasComputed + g.mu.Unlock() + if !ok { + return meshwants.KeepNowAnswer{}, errors.New("no pass has computed an assignment graph revision yet") + } + g.keepMu.Lock() + defer g.keepMu.Unlock() + p.logf("the keeping of the assignment graph runs now by hand: %s", in.Why) + if _, err := p.keepRevisionOf(ctx, c); err != nil { + return meshwants.KeepNowAnswer{Fault: err.Error()}, nil + } + rev, _, _ := g.keptNow() + return meshwants.KeepNowAnswer{Revision: &rev}, nil +} + +// keepNowVerb is the process's handler of the seat mesh-wants' act keep-now. +func keepNowVerb(ctx context.Context, p *passer) map[string]process.SeatHandler { + return map[string]process.SeatHandler{ + meshwants.VerbKeepNow: func(_ string, raw json.RawMessage) (any, error) { + var in meshwants.KeepNowInput + if err := json.Unmarshal(raw, &in); err != nil { + return nil, fmt.Errorf("the arguments are not JSON: %w", err) + } + return p.keepNow(ctx, in) + }, + } } // graphInputs are what the pass's assignment graph is computed from: every followed branch's catalog half, the placements, each @@ -366,3 +450,63 @@ func (p *passer) graphKept() (meshwants.AssignmentRevision, int64, bool) { } return p.graph.keptNow() } + +// keepingFailed marks the keeping of c wanted again after keeping-retry (design 47 H2 §813 (2)): run then unless a newer +// revision was handed meanwhile, whose own keeping runs. The third failure in a row raises keeping-failed (scope module). +func (p *passer) keepingFailed(ctx context.Context, c assigngraph.Computed, err error) { + g := p.graph + g.mu.Lock() + g.failures++ + n, retry, after := g.failures, g.retry, g.after + g.mu.Unlock() + if retry <= 0 { + retry = defaultKeepingRetry + } + if after == nil { + after = func(d time.Duration, f func()) { time.AfterFunc(d, f) } + } + p.logf("the assignment graph revision is not kept (%d in a row), so no want is said as wanted until it is; kept again in %s: %v", n, retry, err) + if n == keepingFailedAt && p.findings.Put != nil { + if err := p.findings.Put(keepingFailedFinding(n, err).Finding(p.clock())); err != nil { + p.logf("the finding %s is not written: %v", kindKeepingFailed, err) + } + } + after(retry, func() { + g.mu.Lock() + newer := g.handed && !assigngraph.Same(g.handedRev, c.Revision) + if !newer { + g.handedRev, g.handed = c.Revision, true + } + g.mu.Unlock() + if !newer { + p.needKeeping(ctx, keepNeed{c: &c}) + } + }) +} + +// keepingSucceeded ends a run of failures: keeping-failed, raised at the third, is cleared. +func (p *passer) keepingSucceeded() { + g := p.graph + g.mu.Lock() + n := g.failures + g.failures = 0 + g.mu.Unlock() + if n >= keepingFailedAt && p.findings.Delete != nil { + o := keepingFailedFinding(n, nil) + if err := p.findings.Delete(views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)); err != nil { + p.logf("the finding %s is not deleted: %v", kindKeepingFailed, err) + } + } +} + +// keepingFailedFinding is keeping-failed of the assigner on this node, after n failures in a row (design 47 H2 §813 (2)). +func keepingFailedFinding(n int, err error) assign.Observed { + summary := fmt.Sprintf("the keeping of the assignment graph revisions failed %d times in a row", n) + if err != nil { + summary += ": " + err.Error() + } + return assign.Observed{Scope: mcc.ScopeModule, ID: assignment.Module() + "." + thisNode, Kind: kindKeepingFailed, + Severity: mcseat.FindingSeverityWarning, Summary: summary, Headline: "The assignment graph is not kept", + Explanation: summary + ". No want is said as wanted until a keeping succeeds; it is run again after keeping-retry.", + Source: "assigner." + kindKeepingFailed} +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go index 5acdb4241..d75874ac6 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "slices" "strings" @@ -11,6 +12,7 @@ import ( "time" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -18,6 +20,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/catalogue" + "mesh-delivery-assigner/internal/assign" "mesh-delivery-assigner/internal/assigngraph" "mesh-delivery-assigner/internal/buckets" ) @@ -273,4 +276,65 @@ func TestAPassWritesWantsWhileAKeepingIsBlocked(t *testing.T) { if !said("the pass took ") { t.Error("the pass does not say its duration") } + open() + kept.Wait() + // Design 47 H2 §813 (2): a keeping that fails runs again after keeping-retry with no pass, raising keeping-failed at the third + // failure in a row, cleared at the next that succeeds. + t.Run("a failed keeping runs again", func(t *testing.T) { + failing := &failingObjects{blockedObjects: objects, fail: 3} + var retries []time.Duration + var put []mcseat.Finding + var deleted []string + p.findings = assign.Findings{Put: func(f mcseat.Finding) error { put = append(put, f); return nil }, + Delete: func(key string) error { deleted = append(deleted, key); return nil }} + p.graph.objects, p.graph.background, p.graph.retry = failing, nil, 30*time.Second + p.graph.after = func(d time.Duration, f func()) { retries = append(retries, d); f() } + placeOn(t, p.w, "shanks", "web") // new inputs: a keeping is handed + pass() + if len(retries) != 3 || retries[0] != 30*time.Second || len(put) != 1 || put[0].Kind != kindKeepingFailed || put[0].Scope != "module" || + len(deleted) != 1 || !strings.Contains(deleted[0], kindKeepingFailed) || p.graph.failures != 0 { + t.Fatalf("retries %v, raised %+v, cleared %v, failures %d", retries, put, deleted, p.graph.failures) + } + if _, _, ok := p.graph.keptNow(); !ok || failing.fail != 0 { + t.Fatal("the revision is not kept once the store takes it") + } + // keep-now (H-A-417): the keeping by hand answers the revision kept, or the fault met; it needs a why. + if _, err := p.keepNow(context.Background(), meshwants.KeepNowInput{}); err == nil { + t.Error("a keeping by hand with no why runs") + } + if a, err := p.keepNow(context.Background(), meshwants.KeepNowInput{Why: "the store was full"}); err != nil || a.Revision == nil || a.Fault != "" { + t.Errorf("keep-now answered %+v, %v; want the revision kept", a, err) + } + failing.fail, retries = 2, nil + p.graph.after = func(d time.Duration, _ func()) { retries = append(retries, d) } + placeOn(t, p.w, "novox", "web") // a new revision, whose keeping the store refuses at the pass and again by hand + pass() + if a, err := p.keepNow(context.Background(), meshwants.KeepNowInput{Why: "again"}); err != nil || a.Revision != nil || !strings.Contains(a.Fault, "the store answers nothing") || len(retries) != 2 { + t.Errorf("keep-now on a refusing store answered %+v, %v, retries %v; want the fault and a retry", a, err, retries) + } + var asked []string + tool := keepNowTools(func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + asked = append(asked, key+" "+string(raw)) + return json.RawMessage(`{}`), nil + })[0] + if _, err := tool.Run(map[string]any{"why": "by hand"}); err != nil || len(asked) != 2 || !strings.HasPrefix(asked[0], "seat:mesh-hand-acts.hand-act ") || + asked[1] != `seat:mesh-wants.keep-now {"why":"by hand"}` { + t.Errorf("assigner_keep_now: %v, asked %v", err, asked) + } + }) +} + +// failingObjects refuses the next fail puts, then is objects. +type failingObjects struct { + *blockedObjects + fail int +} + +func (f *failingObjects) Put(ctx context.Context, name string, data []byte) error { + if f.fail > 0 { + f.fail-- + return errors.New("the store answers nothing") + } + return f.blockedObjects.Put(ctx, name, data) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index cc6e8168b..71c2b5d21 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -619,7 +619,11 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, objects := newBusObjects(js, process.Account{Node: thisNode, Module: assignment.Module()}, assigngraph.RevisionsObjects) // The keeping and removing of revisions runs off the pass loop, and a keeping that changed what the pass reads wakes // one (design 47 H-A-259). - p.graph = &graphKeeper{objects: objects, halves: iv.h.views, background: func(f func()) { go f() }, wake: wake, + retry, err := keepingRetry() + if err != nil { + return err + } + p.graph = &graphKeeper{objects: objects, halves: iv.h.views, background: func(f func()) { go f() }, wake: wake, retry: retry, reads: newBusObjects(js, process.Account{Node: thisNode, Module: assignment.Module()}, assigngraph.RevisionsObjects)} keep := func(ctx context.Context, ev meshcatalog.CatalogRevisionEvent) error { return p.keepCandidate(ctx, iv.call, ev) @@ -706,6 +710,9 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } verbs := compositionVerbs(runs, steps, record) maps.Copy(verbs, dropVerb(steps)) // the seat's act drop (§778, H-A-404) + if p.graph != nil { + maps.Copy(verbs, keepNowVerb(ctx, p)) // the keeping by hand (§813 (2), H-A-417) + } stopVerbs, err := e.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), verbs) if err != nil { logger.Printf("the verbs %s, %s and %s are not answered until the assigner starts again: %v", meshwants.VerbCompositions, meshwants.VerbCompositionsDrop, meshwants.VerbDrop, err) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go index 94437bcb3..309e0395e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go @@ -418,6 +418,7 @@ func serveTools() error { tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...) tools = append(tools, wantDropTools(runtimeStore{}, stdio.Ask)...) tools = append(tools, asksTools(stdio.QueueWaiting, stdio.State(process.TakenState), stdio.Ask)...) // the seat's work queue (§777) + tools = append(tools, keepNowTools(stdio.Ask)...) // the keeping by hand (§813 (2)) return stdio.Serve("", tools) } diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 1bdb3c35d..af6afbc90 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -93,7 +93,7 @@ func TestManifest(t *testing.T) { // revision, entry and difference from its revisions kept as objects in the old section, form 1, capped at 512 MiB. if len(m.Claims) != 2 || m.Claims[0].Name != meshmoduleassigner.Seat || m.Claims[0].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[0].Serves, []string{"assignments", "proposals", "propose", "withdraw", "again", "placement"}) || - m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input", "compositions", "compositions-drop", "drop"}) { + m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input", "compositions", "compositions-drop", "keep-now", "drop"}) { t.Errorf("the assigner claims %+v, want mesh-module-assigner with its five verbs and mesh-wants with its five", m.Claims) } if len(m.Objects) != 1 || m.Objects[0].Name != "assignment-revisions" || m.Objects[0].MaxMiB != 512 || m.Objects[0].Form != 1 { @@ -105,7 +105,7 @@ func TestManifest(t *testing.T) { // which lists and asks again what each node's engine reads. if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_waits", "assigner_inputs", "assigner_inputs_drop", "assigner_compositions", "assigner_compositions_drop", "assignments", "seat-holders", "wants", "asks", "assigner_read_assignments", "assigner_read_seat_holders", "assigner_read_wants", "assigner_read_findings", "assigner_read_differences", "assigner_read_copies", - "assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_want_drop", "assigner_asks", "assigner_ask_drop"}) { + "assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_want_drop", "assigner_asks", "assigner_ask_drop", "assigner_keep_now"}) { t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads, the seat list's two (ADR 0636 §7) assigner_ask_again (ADR 0668 §5) and assigner_why (design 47 G A-439)", m.Tools) } if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs { @@ -172,11 +172,11 @@ func TestManifest(t *testing.T) { } file, _ := os.ReadFile("module.json") if err := json.Unmarshal(file, &raw); err != nil || raw.Config["outside-wait-minutes"].Default != float64(30) || raw.Config["outside-settle-seconds"].Default != float64(90) || - raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 14 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) || + raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 15 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) || raw.Config["composition-runs-cap-mib"].Default != float64(16) || raw.Config["composition-runs-kept"].Default != "24h" || raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) || // design 47 H2 §775, H-A-381 (2): the taken key's bound and the kept-call pace, runtime settings - raw.Config["ask-taken-bound"].Default != "10m" || raw.Config["kept-call-pace-first"].Default != "1s" || raw.Config["kept-call-pace-max"].Default != "30s" || raw.Config["compose-ask-bound"].Default != "10s" { + raw.Config["ask-taken-bound"].Default != "10m" || raw.Config["kept-call-pace-first"].Default != "1s" || raw.Config["kept-call-pace-max"].Default != "30s" || raw.Config["compose-ask-bound"].Default != "10s" || raw.Config["keeping-retry"].Default != "30s" { t.Errorf("config = %+v (%v), want outside-wait-minutes of 30, outside-settle-seconds of 90 and ask-operator-for of take-off,remove-node,stops (A-425a)", raw.Config, err) } for _, f := range []struct{ env, path, key string }{ diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 699f5dfd1..3bb05e6ef 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -28,6 +28,7 @@ "input", "compositions", "compositions-drop", + "keep-now", "drop" ] } @@ -132,7 +133,8 @@ "assigner_why", "assigner_want_drop", "assigner_asks", - "assigner_ask_drop" + "assigner_ask_drop", + "assigner_keep_now" ], "state": [ { "name": "assignments", "history": 64 }, @@ -245,6 +247,13 @@ "default": "10s", "runtime": true }, + "keeping-retry": { + "type": "string", + "description": "how long after a failed keeping of the assignment graph revisions the assigner keeps them again; keeping-failed is raised at the third failure in a row, a Go duration such as 30s (design 47 H2 §813 (2)).", + "trusted": false, + "default": "30s", + "runtime": true + }, "kept-call-pace-first": { "type": "string", "description": "the first wait before a kept call met by a transport fault is made again, doubled per call up to kept-call-pace-max, a Go duration (design 47 H2 §775).", diff --git a/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go b/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go index e83bbf8e2..b13b8b95d 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go @@ -20,7 +20,7 @@ const Form = 2 // whose fields changed while its forms did not (design 47 H2 §780). const ( generatedForms = "2" - generatedFieldsDigest = "Ask=6cb8b82e2fbe AskTakenEvent=6d5d3da7ac39 AsksByAssignmentsProposalValue=0bb5682c7b3b AsksByBackupNodeModuleFingerprintValue=0bb5682c7b3b AsksByHoldModuleFingerprintValue=0bb5682c7b3b AsksByOutsideNodeModuleFingerprintValue=0bb5682c7b3b AsksByRecordNodeModuleDigestValue=0bb5682c7b3b AsksByRestoreNodeModuleFingerprintValue=0bb5682c7b3b AsksByRevertNodeModuleFingerprintValue=0bb5682c7b3b AssignmentEntry=7d2549782eb2 AssignmentRevision=72c8701f7314 AssignmentRevisionCapabilitiesItem=0c89b0717762 AssignmentRevisionEvent=7faff968ce44 AssignmentRevisionsByCandidateGroupHeadsValue=8553b390118d AssignmentRevisionsByEntryDigestValue=4e5ee7a23e5b AssignmentRevisionsByTrunkRevisionValue=8553b390118d BusAnswer=7538f2f5e68e CatalogOf=7a4646fdbea7 Change=28d1f3c5cd32 ChangeAfterItem=30d718c21581 CompositionReader=5a943c1f8ef1 CompositionRun=a64bc503c99a CompositionRunProgram=5a1202df3497 CompositionsAnswer=f8bca93c7f9c CompositionsByGroupHeadsValue=9428965fab8f CompositionsDropAnswer=52803700282a Difference=4011cff516b1 DifferenceAnswer=629237aa39f0 DifferenceEvent=49066d0dff6d DifferenceRefusedItem=0e9caa83ace7 DifferencesByNodeModuleValue=a22487ffadfd DropAnswer=7538f2f5e68e Edge=a71709025a08 EntryAnswer=05adebad70d6 Fill=80f442916429 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 Held=15abbd6a1a33 HeldCandidate=04872f6afbfa InputAnswer=cd41ee65565d NodeEntryRef=3888ae847653 NotRunning=279b628db0ec PipelineWanted=e1e379c21e59 PipelineWantedEvent=59946b31ab7d PolicyChangedEvent=7538f2f5e68e RevisionAnswer=39002c770cbd RoomEvent=5bd4df78ed84 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 StepAccept=18efeefb6c34 StepEnded=edd08ab1a18b StepEndedEvent=207787118dac StepProgressEvent=8215e490ea8d StepRun=b975f402601a UpgradeAnswer=7538f2f5e68e Wait=951500be820c WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 WantedEvent=58cf7a9f646b WantsByHeldModuleFingerprintValue=ff14aa998a5d WantsByWantNodeModuleValue=ab7c6799fd5d WantsByWantNodeNodeValue=a737a2108d53" + generatedFieldsDigest = "Ask=6cb8b82e2fbe AskTakenEvent=6d5d3da7ac39 AsksByAssignmentsProposalValue=0bb5682c7b3b AsksByBackupNodeModuleFingerprintValue=0bb5682c7b3b AsksByHoldModuleFingerprintValue=0bb5682c7b3b AsksByOutsideNodeModuleFingerprintValue=0bb5682c7b3b AsksByRecordNodeModuleDigestValue=0bb5682c7b3b AsksByRestoreNodeModuleFingerprintValue=0bb5682c7b3b AsksByRevertNodeModuleFingerprintValue=0bb5682c7b3b AssignmentEntry=7d2549782eb2 AssignmentRevision=72c8701f7314 AssignmentRevisionCapabilitiesItem=0c89b0717762 AssignmentRevisionEvent=7faff968ce44 AssignmentRevisionsByCandidateGroupHeadsValue=8553b390118d AssignmentRevisionsByEntryDigestValue=4e5ee7a23e5b AssignmentRevisionsByTrunkRevisionValue=8553b390118d BusAnswer=7538f2f5e68e CatalogOf=7a4646fdbea7 Change=28d1f3c5cd32 ChangeAfterItem=30d718c21581 CompositionReader=5a943c1f8ef1 CompositionRun=a64bc503c99a CompositionRunProgram=5a1202df3497 CompositionsAnswer=f8bca93c7f9c CompositionsByGroupHeadsValue=9428965fab8f CompositionsDropAnswer=52803700282a Difference=4011cff516b1 DifferenceAnswer=629237aa39f0 DifferenceEvent=49066d0dff6d DifferenceRefusedItem=0e9caa83ace7 DifferencesByNodeModuleValue=a22487ffadfd DropAnswer=7538f2f5e68e Edge=a71709025a08 EntryAnswer=05adebad70d6 Fill=80f442916429 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 Held=15abbd6a1a33 HeldCandidate=04872f6afbfa InputAnswer=cd41ee65565d KeepNowAnswer=9962ab9fc051 NodeEntryRef=3888ae847653 NotRunning=279b628db0ec PipelineWanted=e1e379c21e59 PipelineWantedEvent=59946b31ab7d PolicyChangedEvent=7538f2f5e68e RevisionAnswer=39002c770cbd RoomEvent=5bd4df78ed84 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 StepAccept=18efeefb6c34 StepEnded=edd08ab1a18b StepEndedEvent=207787118dac StepProgressEvent=8215e490ea8d StepRun=b975f402601a UpgradeAnswer=7538f2f5e68e Wait=951500be820c WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 WantedEvent=58cf7a9f646b WantsByHeldModuleFingerprintValue=ff14aa998a5d WantsByWantNodeModuleValue=ab7c6799fd5d WantsByWantNodeNodeValue=a737a2108d53" ) const ( @@ -32,6 +32,7 @@ const ( VerbInput = "input" VerbCompositions = "compositions" VerbCompositionsDrop = "compositions-drop" + VerbKeepNow = "keep-now" VerbDrop = "drop" ) @@ -57,7 +58,7 @@ const ( // Verbs is every verb the seat serves, in its definition's order: a new copy each call. func Verbs() []string { - return []string{VerbUpgrade, VerbBus, VerbRevision, VerbEntry, VerbDifference, VerbInput, VerbCompositions, VerbCompositionsDrop, VerbDrop} + return []string{VerbUpgrade, VerbBus, VerbRevision, VerbEntry, VerbDifference, VerbInput, VerbCompositions, VerbCompositionsDrop, VerbKeepNow, VerbDrop} } // Events is every event published on the seat's subject: a new copy each call. @@ -1246,6 +1247,22 @@ type CompositionsDropAnswer struct { Run CompositionRun `json:"run"` } +// KeepNowInput is what keep-now takes. +type KeepNowInput struct { + // why the keeping runs now + Why string `json:"why"` +} + +// KeepNowAnswer is what keep-now answers. +type KeepNowAnswer struct { + // the fault the keeping met; absent when it kept the revision + Fault string `json:"fault,omitempty"` + // is the form of what this value carries, left out while it is 1 (design 47 H2 §780). + Form int64 `json:"form,omitempty"` + // the revision kept + Revision *AssignmentRevision `json:"revision,omitempty"` +} + // DropInput is what drop takes. type DropInput struct { // who drops it @@ -1557,6 +1574,15 @@ func DecodeCompositionsDropAnswer(raw []byte) (CompositionsDropAnswer, int, erro return v, formRead(raw), nil } +// DecodeKeepNowAnswer reads a KeepNowAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780). +func DecodeKeepNowAnswer(raw []byte) (KeepNowAnswer, int, error) { + var v KeepNowAnswer + if err := json.Unmarshal(raw, &v); err != nil { + return v, 0, err + } + return v, formRead(raw), nil +} + // DecodeDropAnswer reads a DropAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780). func DecodeDropAnswer(raw []byte) (DropAnswer, int, error) { var v DropAnswer diff --git a/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json b/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json index e13f7058b..3b72c9c4e 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json @@ -278,6 +278,36 @@ ] } }, + { + "name": "keep-now", + "description": "Keep the assignment graph's revisions now, by hand (design 47 H2 §813 (2); H-A-417): written in the hand-act log first by the module's tool assigner_keep_now, then the keeping runs at once on the pass's newest computed revision; answers the revision kept, or the fault the keeping met.", + "input": { + "type": "object", + "properties": { + "why": { + "type": "string", + "description": "why the keeping runs now" + } + }, + "required": [ + "why" + ] + }, + "answer": { + "type": "object", + "properties": { + "revision": { + "$ref": "#/defs/AssignmentRevision", + "description": "the revision kept" + }, + "fault": { + "type": "string", + "description": "the fault the keeping met; absent when it kept the revision" + } + } + }, + "class": "act" + }, { "name": "drop", "description": "End an ask on the seat's work queue for a caller that no longer waits for it (its step ended by its bound): one waiting is ended dropped when it is pulled, one taken is cancelled; both end the step dropped, naming who dropped it and why, so no holder works on it after (design 47 H2 §778; ADR 0664 §4).", diff --git a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts index b9f4ef056..dd3efa85d 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts @@ -13,6 +13,7 @@ export declare const verbs: { readonly input: "input"; readonly compositions: "compositions"; readonly compositionsDrop: "compositions-drop"; + readonly keepNow: "keep-now"; readonly drop: "drop"; }; @@ -672,6 +673,22 @@ export interface CompositionsDropAnswer { "run": CompositionRun; } +/** is what keep-now takes. */ +export interface KeepNowInput { + /** why the keeping runs now */ + "why": string; +} + +/** is what keep-now answers. */ +export interface KeepNowAnswer { + /** the fault the keeping met; absent when it kept the revision */ + "fault"?: string; + /** is the form of what this value carries, left out while it is 1 (design 47 H2 §780). */ + "form"?: number; + /** the revision kept */ + "revision"?: AssignmentRevision; +} + /** is what drop takes. */ export interface DropInput { /** who drops it */ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js index 7c9016b74..94fb35ed8 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js @@ -13,6 +13,7 @@ export const verbs = { input: "input", compositions: "compositions", compositionsDrop: "compositions-drop", + keepNow: "keep-now", drop: "drop", }; From 1aba89e8ba3aa2281193ac64c2821e62bc5d52d1 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:15:54 +0100 Subject: [PATCH 05/12] A6: a validate step counts only problems new against its case before MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A take-in that touched a manifest failed its validate step on every problem that manifest already had, so an old problem blocked any change to it. The step now counts, for every manifest and between manifests, only the problems the case before does not hold; assigner_validate_before (pipeline, module) answers from the kept validate run the ones it holds and did not count. Builds: design 47 H2 §814 --- .../cmd/mesh-delivery-assigner/tools.go | 3 +- .../mesh-delivery-assigner/validatestep.go | 114 +++++++++++------- .../validatestep_test.go | 20 ++- .../mesh-delivery-assigner/manifest_test.go | 2 +- modules/mesh-delivery-assigner/module.json | 3 +- 5 files changed, 94 insertions(+), 48 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go index 309e0395e..eb704a538 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go @@ -418,7 +418,8 @@ func serveTools() error { tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...) tools = append(tools, wantDropTools(runtimeStore{}, stdio.Ask)...) tools = append(tools, asksTools(stdio.QueueWaiting, stdio.State(process.TakenState), stdio.Ask)...) // the seat's work queue (§777) - tools = append(tools, keepNowTools(stdio.Ask)...) // the keeping by hand (§813 (2)) + tools = append(tools, keepNowTools(stdio.Ask)...) + tools = append(tools, validateBeforeTools(runtimeStore{})...) // what a validate step did not count (§814) // the keeping by hand (§813 (2)) return stdio.Serve("", tools) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep.go index 7d3bacc8f..d7bd437c1 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep.go @@ -15,6 +15,9 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-assigner/internal/buckets" ) // The step validate (design 47 H §745; H-A-34, H-A-51, H-A-108): the assigner serves it as an accepted kind `step` of the seat @@ -23,8 +26,8 @@ import ( // reads: the cycle the revision's entries make (graph.RefusedCycle) ends it at once; else the manifest validator built from // the revision's own entry of the module holding the seat mesh-catalog, on the Go build seat as the composer is, runs in the // sandbox over the revision's manifests and its base's (H-A-154), its input kept and served on mesh-wants' input. -// Every mover's step ends the same: failed `cycle` naming the cycle, failed `manifest-refused` naming every problem of a -// manifest the revision alters or between manifests one it alters takes part in, or passed. +// Every mover's step ends the same: failed `cycle` naming the cycle, failed `manifest-refused` naming every problem the case +// before does not hold (design 47 H2 §814; assigner_validate_before answers the ones it holds), or passed. const ( // validatorPackage is the manifest validator's import path in the module holding the seat mesh-catalog (the indexer's @@ -112,16 +115,12 @@ func (r *compositionRun) judgeValidate(ctx context.Context, step meshwants.StepR if err != nil { return couldNotRun("the step's catalog revision is not read: " + err.Error()) } - before, err := r.before(ctx, step, half) - if err != nil { - return couldNotRun("the base of the case before is not read: " + err.Error()) - } // A revision that names no base (the branch's first take-in) is judged against an empty case before, and says so (H-A-151). empty := "" if half.Revision.Base <= 0 { empty = emptyBeforeWords } - if problems := causedProblems(v.Change, v.Before, altered(half, before)); len(problems) > 0 { + if problems := causedProblems(v.Change, v.Before); len(problems) > 0 { said := reasonManifestRefused + ": " + strings.Join(problems, "; ") if empty != "" { said += " (" + empty + ")" @@ -173,60 +172,44 @@ func (r *compositionRun) before(ctx context.Context, step meshwants.StepRun, hal return r.revision(ctx, base) } -// altered are the modules whose manifest the revision adds or changes against the one before it. -func altered(half, before mcc.Half) []string { - var out []string - for name, e := range half.Entries { - if len(e.Manifest) == 0 { - continue - } - b, ok := before.Entries[name] - switch { - case !ok || len(b.Manifest) == 0: - out = append(out, name) - case e.ManifestDigest != "" && b.ManifestDigest != "": - if e.ManifestDigest != b.ManifestDigest { - out = append(out, name) - } - default: - x, _ := json.Marshal(e.Manifest) - y, _ := json.Marshal(b.Manifest) - if string(x) != string(y) { - out = append(out, name) - } - } - } - slices.Sort(out) - return out +// causedProblems are the problems the revision causes (ADR 0578 §2; design 47 H2 §814): of every manifest, and between +// manifests, each problem the case before did not have; a problem the case before holds is never counted, whether the +// revision alters that manifest or not (assigner_validate_before answers those). +func causedProblems(change, before composer.CaseVerdict) []string { + return problemsBy(change, before, false) } -// causedProblems are the problems the revision causes (ADR 0578 §2): every problem of a manifest it alters, and of any other -// manifest, or between manifests, each problem the case before did not have. -func causedProblems(change, before composer.CaseVerdict, touched []string) []string { +// uncountedProblems are the change case's problems the case before holds too, which causedProblems does not count (§814). +func uncountedProblems(change, before composer.CaseVerdict) []string { + return problemsBy(change, before, true) +} + +// problemsBy are the change case's problems, per manifest, that the case before had (had) or did not have (!had). +func problemsBy(change, before composer.CaseVerdict, had bool) []string { var out []string for _, module := range slices.Sorted(maps.Keys(change.Manifests)) { read := change.Manifests[module] if read.Accepted || read.Refused == "" { continue } - had := map[string]bool{} + held := map[string]bool{} for _, p := range strings.Split(before.Manifests[module].Refused, "\n") { - had[p] = true + held[p] = true } - var caused []string + var named []string for _, p := range strings.Split(read.Refused, "\n") { - if p != "" && (slices.Contains(touched, module) || !had[p]) { - caused = append(caused, p) + if p != "" && held[p] == had { + named = append(named, p) } } - if len(caused) == 0 { + if len(named) == 0 { continue } what := module if module == betweenManifests { what = "between manifests" } - out = append(out, what+": "+strings.Join(caused, "; ")) + out = append(out, what+": "+strings.Join(named, "; ")) } return out } @@ -380,3 +363,50 @@ func validateRunVerdict(answer []byte) (map[string]any, string) { } return verdict, "" } + +// validateBeforeTool is the tool that answers what a validate step did not count (design 47 H2 §814). +const validateBeforeTool = "assigner_validate_before" + +// validateBefore answers, for the validate run of pipeline, the problems of the change case that its case before holds too, +// which the step did not count, for module alone when one is named; read from the run the state composition-runs keeps. +func validateBefore(st store, pipeline, module string) (map[string]any, error) { + key, err := validateKeyOf(meshwants.StepRun{Pipeline: strings.TrimSpace(pipeline)}) + if err != nil { + return nil, err + } + raw, _, ok, err := st.get(buckets.CompositionRunsState, key) + if err != nil { + return nil, fmt.Errorf("the state %s is not read: %w", buckets.CompositionRunsState, err) + } + if !ok { + return nil, fmt.Errorf("no validate run %s is kept", key) + } + var run meshwants.CompositionRun + if err := json.Unmarshal(raw, &run); err != nil { + return nil, fmt.Errorf("the validate run %s does not read: %w", key, err) + } + var v validateVerdict + if b, err := json.Marshal(run.Verdict); err != nil || json.Unmarshal(b, &v) != nil { + return nil, fmt.Errorf("the validate run %s holds no verdict that reads", key) + } + change, before := v.Change, v.Before + if module = strings.TrimSpace(module); module != "" { + change = composer.CaseVerdict{Name: change.Name, Manifests: map[string]composer.ManifestRead{module: change.Manifests[module]}} + } + uncounted := uncountedProblems(change, before) + if uncounted == nil { + uncounted = []string{} + } + return map[string]any{"run": key, "outcome": run.Outcome, "uncounted": uncounted}, nil +} + +// validateBeforeTools are the module's tool of the validate step's before case. +func validateBeforeTools(st store) []stdio.Tool { + return []stdio.Tool{{Name: validateBeforeTool, Description: "The problems a validate step did not count (design 47 H2 §814): of the " + + "validate run of a pipeline's take-in revision, every problem of the change case its case before holds too, per manifest, " + + "for one module when named. Read-only.", + Input: schema(map[string]string{"pipeline": "the pipeline's key, .", "module": "one module; absent: every one"}, []string{"pipeline"}), + Run: func(args map[string]any) (any, error) { + return validateBefore(st, arg(args, "pipeline"), arg(args, "module")) + }}} +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go index 0326f4f19..4d7985b13 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "slices" "strings" "sync" "testing" @@ -14,6 +15,8 @@ import ( meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" + + "mesh-delivery-assigner/internal/buckets" ) // A10/AC1, A10/AC2 (design 47 H §745; H-A-34, H-A-51, H-A-108): every mover's validate step of one take-in reads one validate @@ -86,8 +89,8 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) { asks, requests = append(asks, a), append(requests, req) mu.Unlock() answer, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{ - {Name: changeCase, Manifests: map[string]composer.ManifestRead{"web": {Refused: `unknown field "colour"`}, "db": {Refused: "an old problem"}, "indexer": {Accepted: true}}}, - {Name: beforeCase, Manifests: map[string]composer.ManifestRead{"db": {Refused: "an old problem"}}}}}) + {Name: changeCase, Manifests: map[string]composer.ManifestRead{"web": {Refused: "unknown field \"colour\"\nan inherited problem"}, "db": {Refused: "an old problem"}, "indexer": {Accepted: true}}}, + {Name: beforeCase, Manifests: map[string]composer.ManifestRead{"db": {Refused: "an old problem"}, "web": {Refused: "an inherited problem"}}}}}) go func() { if err := runs.heardEnd(buildstep.StepEnded{Asker: runAsker, Ask: a.ID, Outcome: buildstep.OutcomePassed, Node: "novox"}, func(buildstep.StepEnded) ([]byte, error) { return answer, nil }); err != nil { @@ -115,10 +118,21 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) { wg.Wait() for i, o := range outs { if o.Outcome != meshwants.StepEndedOutcomeFailed || !strings.HasPrefix(o.Said, reasonManifestRefused+": ") || !strings.Contains(o.Said, `web: unknown field "colour"`) || - strings.Contains(o.Said, "an old problem") { + strings.Contains(o.Said, "an old problem") || strings.Contains(o.Said, "an inherited problem") { // H2 §814: only new problems t.Errorf("mover %d's validate step ended %+v", i, o) } } + // assigner_validate_before (H2 §814): the problems the before case holds, which the step did not count. + runsKV := memStore{buckets.CompositionRunsState: &memKV{values: map[string][]byte{}}} + kept, _, _, _ := runs.read("main.7" + validateKeySuffix) + runsKV[buckets.CompositionRunsState].values["main.7"+validateKeySuffix], _ = json.Marshal(kept) + before, err := validateBeforeTools(runsKV)[0].Run(map[string]any{"pipeline": "main.7", "module": "web"}) + if b, _ := before.(map[string]any); err != nil || !slices.Equal(b["uncounted"].([]string), []string{"web: an inherited problem"}) { + t.Errorf("assigner_validate_before answered %+v, %v", before, err) + } + if _, err := validateBeforeTools(runsKV)[0].Run(map[string]any{"pipeline": "main.6"}); err == nil { + t.Error("assigner_validate_before answers a run not kept") + } if len(asks) != 1 { t.Fatalf("%d validator asks were put for one take-in, want 1", len(asks)) } diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index af6afbc90..dee553321 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -105,7 +105,7 @@ func TestManifest(t *testing.T) { // which lists and asks again what each node's engine reads. if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_waits", "assigner_inputs", "assigner_inputs_drop", "assigner_compositions", "assigner_compositions_drop", "assignments", "seat-holders", "wants", "asks", "assigner_read_assignments", "assigner_read_seat_holders", "assigner_read_wants", "assigner_read_findings", "assigner_read_differences", "assigner_read_copies", - "assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_want_drop", "assigner_asks", "assigner_ask_drop", "assigner_keep_now"}) { + "assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_want_drop", "assigner_asks", "assigner_ask_drop", "assigner_keep_now", "assigner_validate_before"}) { t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads, the seat list's two (ADR 0636 §7) assigner_ask_again (ADR 0668 §5) and assigner_why (design 47 G A-439)", m.Tools) } if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs { diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 3bb05e6ef..2a2479559 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -134,7 +134,8 @@ "assigner_want_drop", "assigner_asks", "assigner_ask_drop", - "assigner_keep_now" + "assigner_keep_now", + "assigner_validate_before" ], "state": [ { "name": "assignments", "history": 64 }, From 8f89dd29f51a6b35f66c9a2f44e852b582bd9a34 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:19:20 +0100 Subject: [PATCH 06/12] A2: the assigner reads an oversized run answer through the build seat's verb answer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The build-step part's output verb loses call and chunks (form 2): a composition or validate run's answer over the inline bound is read through the Go build seat's read verb answer (buildstep.StepEnded.ReadOutput, the chunked call), so the manifest invokes seat:node-build-go.answer in place of seat:node-build-go.output. Builds: design 47 H2 §791; H-A-412 and its addendum --- modules/mesh-delivery-assigner/manifest_test.go | 2 +- modules/mesh-delivery-assigner/module.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index dee553321..03bde3065 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -153,8 +153,8 @@ func TestManifest(t *testing.T) { "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone "seat:node-backup.backed-up", "seat:node-backup.now", "seat:node-build-go.accept.check", // A-231 + "seat:node-build-go.answer", // A-254, H-A-412: a composition run's answer over the inline bound, read through the build seat's verb answer "seat:node-build-go.drop", // §720, A-469s: a composition run past its bound drops its ask: the composition run's ask on the Go build seat's queue check (accept.check.) - "seat:node-build-go.output", // A-254: a composition run's answer kept as the build seat's object "seat:node-engine.compose-candidate", // §354, A-226: each machine's composition input "seat:operator-channel.asks"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) { // ADR 0668 §4: the ended asks at bind t.Errorf("invokes = %v, want %v", m.Invokes, want) diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 2a2479559..097af3f1f 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -79,7 +79,7 @@ "seat:mesh-hand-acts.warranted", "seat:node-backup.backed-up", "seat:node-backup.now", - "seat:node-build-go.output", + "seat:node-build-go.answer", "seat:node-build-go.drop", "seat:node-engine.compose-candidate", "seat:operator-channel.asks", From 477d351fc0fef889e6a7806ab9de57cb41b80b48 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:19:20 +0100 Subject: [PATCH 07/12] A4: a want carries its run and is written by compare-and-set refusing a lower one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two writers of a node's want could leave it at an older delivery, and the node-engine could not tell a newer want from an older one. Each want now carries run: the source revision of the pipeline that released it, or, for a want of no pipeline (a put, a placement's build, a removal), the newest source the assigner holds. It is written by compare-and-set (create, or update at the revision read), refusing a want whose run is lower than the one written, said in the journal; a want that differs only by its run stands, so no node composes again for it. The answer's other half is the node-engine's: its manifest states declaration-reads and declaration-writes, the assigner's neither. Builds: design 47 H2 §813 (1); H-A-355 (1), H-A-416 --- .../mesh-delivery-assigner/graphkept_test.go | 24 ++++++ .../cmd/mesh-delivery-assigner/pass.go | 80 ++++++++++++++++++- .../cmd/mesh-delivery-assigner/pass_test.go | 2 + 3 files changed, 105 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go index d75874ac6..857622492 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go @@ -95,6 +95,30 @@ func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) { if err := json.Unmarshal(events[len(events)-1].body, &w); err != nil || len(events) != 3 || !w.Removed || w.Fp != fpTest { t.Errorf("a removing want said %+v (%v), %d events", w, err, len(events)) } + // Design 47 H2 §813 (1), H-A-416: a want carries its run, its pipeline's source in decimal, or the newest source held for a + // want of no pipeline; it is written by compare-and-set refusing a lower run, and one that differs only by its run stands. + t.Run("a want's run", func(t *testing.T) { + wkey, _ := mwc.WantKey("g14", "db") + stored := func() mwc.Want { w, _ := mwc.DecodeWant(kv.values[wkey]); return w } + p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1}) + if w := stored(); w.Run != "12" || w.Pipeline != "main.12" { + t.Fatalf("the want of pipeline main.12 is %+v, want run 12", w) + } + puts := kv.puts + p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.1", Pipeline: "main.11", Rank: 1}) + if w := stored(); w.Run != "12" || w.Version != "0.1.2" || kv.puts != puts { + t.Errorf("a want of a lower run replaced %+v (%d writes)", w, kv.puts-puts) + } + p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1}) + if kv.puts != puts { + t.Error("a want that stands was written again") + } + skey, _ := mwc.WantKey("shanks", "db") + p.putWant(ctx, "shanks", "db", mwc.Want{Fp: fpTest, Put: true, Words: "put: by hand"}) + if w, _ := mwc.DecodeWant(kv.values[skey]); w.Run != "9" { + t.Errorf("a put's want is %+v, want the newest source held, 9", w) + } + }) // A-201, A-235: a run's candidate catalog revision, read whole through the seat's verb revision, keeps the run's assignment // graph under candidate.. in place of its branch's half; it says nothing and writes no want, a try's candidate diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 56aec2127..9141378d0 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -7,8 +7,10 @@ import ( "errors" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "mesh-delivery-assigner/internal/buckets" + "reflect" "slices" "sort" + "strconv" "strings" "time" @@ -21,6 +23,8 @@ import ( "git.novox.be/novox/mesh-sdk/go/seatlist" "sync" + "github.com/nats-io/nats.go/jetstream" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -303,11 +307,85 @@ func (p *passer) putWant(ctx context.Context, node, module string, want mwc.Want fp = before.Fp } } - if p.written(ctx, buckets.Wants(), key, func() ([]byte, error) { return mwc.EncodeWant(want) }) { + if p.wantWritten(ctx, node, module, key, want) { p.sayWanted(node, module, fp, want.Pipeline, want.Removed) } } +// runOf is the run a want carries (design 47 H2 §813 (1); H-A-355 (1), H-A-416): the source revision of the pipeline that +// released it, `.`, in decimal; for a want of no pipeline (a put, a placement's build, a removal) the newest +// source the assigner holds, the trunk's source of its kept assignment graph revision; "" when it holds none. +func (p *passer) runOf(want mwc.Want) string { + if i := strings.LastIndexByte(want.Pipeline, '.'); i >= 0 { + if n, err := strconv.ParseInt(want.Pipeline[i+1:], 10, 64); err == nil && n > 0 { + return strconv.FormatInt(n, 10) + } + } + if p.graph != nil { + if _, source, ok := p.graph.keptNow(); ok && source > 0 { + return strconv.FormatInt(source, 10) + } + } + return "" +} + +// runNumber is a want's run as a number, 0 for none. +func runNumber(run string) int64 { + n, _ := strconv.ParseInt(run, 10, 64) + return n +} + +// wantWritten writes the want of module on node under key by compare-and-set, carrying its run (design 47 H2 §813 (1)): a want +// that differs from the one written only by its run is not written again (its run stays, so a node never composes again for +// it), and one whose run is lower than the run written is refused, said in the journal. It answers whether it wrote. +func (p *passer) wantWritten(ctx context.Context, node, module, key string, want mwc.Want) bool { + var held *mwc.Want + if raw := p.w.value(buckets.Wants(), key); raw != nil { + if h, err := mwc.DecodeWant(raw); err == nil { + held = &h + } + } + want.Run = p.runOf(want) + if held != nil { + same := *held + same.Run = want.Run + if reflect.DeepEqual(same, want) { + want.Run = held.Run // it stands: nothing is written + } else if runNumber(held.Run) > runNumber(want.Run) { + p.logf("the want of %s on %s is not written: its run %q is lower than the run %q written (§813 (1))", module, node, want.Run, held.Run) + return false + } + } + value, err := mwc.EncodeWant(want) + if err != nil { + p.logf("%s %s is not written: %v", buckets.Wants(), key, err) + return false + } + if held := p.w.value(buckets.Wants(), key); held != nil && bytes.Equal(held, value) { + return false + } + kv := p.kvs[buckets.Wants()] + entry, err := kv.Get(ctx, key) + switch { + case errors.Is(err, jetstream.ErrKeyNotFound): + _, err = kv.Create(ctx, key, value) + case err == nil: + if h, derr := mwc.DecodeWant(entry.Value()); derr == nil && runNumber(h.Run) > runNumber(want.Run) { + p.logf("the want of %s on %s is not written: its run %q is lower than the run %q written (§813 (1))", module, node, want.Run, h.Run) + return false + } + _, err = kv.Update(ctx, key, value, entry.Revision()) + } + if err != nil { + if !errors.Is(err, context.Canceled) { + p.logf("%s %s could not be written: %v; the next pass writes it again", buckets.Wants(), key, err) + } + return false + } + p.w.put(buckets.Wants(), key, value) + return true +} + // staleAsk says why the hold, restore or record ask at key (read as the SDK reads it) is about something the trunk has left // behind, or "" when it is not: a module no longer on the trunk, a fingerprint that is no longer the module's trunk fingerprint, // or, for a record ask, a composition the module's node no longer waits on. A backup ask, and a revert ask (design 47 G §710), is stale when the module's want on its diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 3a82d04a2..16c9afce9 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -57,6 +57,7 @@ func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error) func (m *memKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) { m.values[key] = value + m.puts++ // a write that reaches the bucket (the wants' compare-and-set, design 47 H2 §813 (1)) return uint64(len(m.values)), nil } @@ -253,6 +254,7 @@ func (m *memKV) Update(_ context.Context, key string, value []byte, revision uin return 0, errors.New("wrong last sequence") } m.values[key] = value + m.puts++ return uint64(len(value)), nil } From b9bc44024de984437808c44c7c991a53a7b40ae8 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:19:20 +0100 Subject: [PATCH 08/12] A2: the seat mesh-wants' drop says the step ends could-not-run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The drop's description said a dropped ask's step ends "dropped", an outcome the step-ended has not: it ends could-not-run, its why naming the drop and who asked it, as the assigner says it. Builds: design 47 H2 §778; H-A-401 --- modules/mesh-delivery-contracts/seats/mesh-wants/seat.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json b/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json index 3b72c9c4e..f9d926694 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json @@ -310,7 +310,7 @@ }, { "name": "drop", - "description": "End an ask on the seat's work queue for a caller that no longer waits for it (its step ended by its bound): one waiting is ended dropped when it is pulled, one taken is cancelled; both end the step dropped, naming who dropped it and why, so no holder works on it after (design 47 H2 §778; ADR 0664 §4).", + "description": "End an ask on the seat's work queue for a caller that no longer waits for it (its step ended by its bound): one waiting is ended dropped when it is pulled, one taken is cancelled; both end the step could-not-run, its why naming the drop and who asked it, so no holder works on it after (design 47 H2 §778; ADR 0664 §4).", "input": { "type": "object", "properties": { From 40ad2650bdffd31fab1cefacd57081885ca482b2 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:22:52 +0100 Subject: [PATCH 09/12] A4: a want of no pipeline carries the newest source held for it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A placement's build or a put written after a delivery carried the kept graph's trunk source as its run, which can be older than the delivered want's, so the compare-and-set refused a person's act (the whole test of a placement back at a module's first build). The newest source the assigner holds for such a want is now the kept trunk source or the run of the want written, whichever is newer. Builds: design 47 H2 §813 (1); H-A-416 --- .../mesh-delivery-assigner/graphkept_test.go | 5 +++++ .../cmd/mesh-delivery-assigner/pass.go | 20 +++++++++++++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go index 857622492..fdf8d4fd0 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go @@ -118,6 +118,11 @@ func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) { if w, _ := mwc.DecodeWant(kv.values[skey]); w.Run != "9" { t.Errorf("a put's want is %+v, want the newest source held, 9", w) } + // a placement's build after the delivery of run 12: the newest source held for it is the want's own, never refused + p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.0"}) + if w := stored(); w.Run != "12" || w.Version != "0.1.0" { + t.Errorf("a placement's want after run 12 is %+v, want it written at run 12", w) + } }) // A-201, A-235: a run's candidate catalog revision, read whole through the seat's verb revision, keeps the run's assignment diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 9141378d0..fcd40fdf1 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -314,19 +314,27 @@ func (p *passer) putWant(ctx context.Context, node, module string, want mwc.Want // runOf is the run a want carries (design 47 H2 §813 (1); H-A-355 (1), H-A-416): the source revision of the pipeline that // released it, `.`, in decimal; for a want of no pipeline (a put, a placement's build, a removal) the newest -// source the assigner holds, the trunk's source of its kept assignment graph revision; "" when it holds none. -func (p *passer) runOf(want mwc.Want) string { +// source the assigner holds for it: the trunk's source of its kept assignment graph revision, or the run of the want written +// (held) when that is newer, so a person's put or placement after a delivery is never refused as older; "" when it holds none. +func (p *passer) runOf(want mwc.Want, held *mwc.Want) string { if i := strings.LastIndexByte(want.Pipeline, '.'); i >= 0 { if n, err := strconv.ParseInt(want.Pipeline[i+1:], 10, 64); err == nil && n > 0 { return strconv.FormatInt(n, 10) } } + var newest int64 if p.graph != nil { - if _, source, ok := p.graph.keptNow(); ok && source > 0 { - return strconv.FormatInt(source, 10) + if _, source, ok := p.graph.keptNow(); ok { + newest = source } } - return "" + if held != nil { + newest = max(newest, runNumber(held.Run)) + } + if newest <= 0 { + return "" + } + return strconv.FormatInt(newest, 10) } // runNumber is a want's run as a number, 0 for none. @@ -345,7 +353,7 @@ func (p *passer) wantWritten(ctx context.Context, node, module, key string, want held = &h } } - want.Run = p.runOf(want) + want.Run = p.runOf(want, held) if held != nil { same := *held same.Run = want.Run From b76b9c58f433483e51087ed375b88c04d92f434e Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:36:34 +0100 Subject: [PATCH 10/12] F2: the bridge's state performed is capped and pruned toward its cap at the bridge's own writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit performed filled on 2026-11-21 with nothing to stop it. It is now bound through mesh-sdk's process.PrunedState: its figures are read when the bridge opens the bucket its binding names, each write is counted, and past prune-mark (70 %) the oldest acts are purged to prune-target (50 %) in one journal line, a key kept under _keep/ staying. The bus drops acts older than three days by the entry's TTL (performed-keep). performed, heads and owed-asks name an 8 MiB cap. The tools bundle serves mesh-delivery-forge-bridge_state_prune_now and _state_keep from stateobjects.PruneTools, each written in the hand-act log. Builds: design 47 H2 §812, §819, H-A-380, H-A-386, H-A-411 (2), H-A-433 --- .../mesh-delivery-forge-bridge-tools/main.go | 24 ++++ .../main_test.go | 42 +++++++ .../cmd/mesh-delivery-forge-bridge/bound.go | 28 ++++- .../integration_test.go | 107 +++++++++++++++++- .../cmd/mesh-delivery-forge-bridge/main.go | 11 +- .../manifest_test.go | 61 ++++++---- .../mesh-delivery-forge-bridge/module.json | 30 ++++- 7 files changed, 273 insertions(+), 30 deletions(-) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go index 560d5659e..a32745934 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go @@ -16,6 +16,8 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/stateobjects" + "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-forge-bridge/internal/bridge" "mesh-delivery-forge-bridge/internal/groups" @@ -126,6 +128,27 @@ func askState(s stdio.KeptState, bucket string, bound time.Duration, done func([ } } +// Module is the module whose tools these are; the generic prune tools are named for it (H-A-380 (3), H-A-433). +const Module = "mesh-delivery-forge-bridge" + +// PerformedPruneTools are `mesh-delivery-forge-bridge_state_prune_now` and `mesh-delivery-forge-bridge_state_keep` over +// the state performed (design 47 H2 §812): a prune to prune-target now, and a key kept from the prune under _keep/ +// until a time, each written in the hand-act log first. +func PerformedPruneTools(performed stdio.Prunable, acts views.HandActs) []stdio.Tool { + return stateobjects.PruneTools(Module, []stateobjects.Pruning{{Name: groups.PerformedState, State: performed}}, acts) +} + +// prunedPerformed is the state performed as the prune tools act on it, with the module's prune-mark and prune-target; a +// setting that is no percentage is said, and the defaults hold. +func prunedPerformed() *stdio.PrunedState { + p, err := stdio.PruneSettings() + if err != nil { + fmt.Fprintf(os.Stderr, "[mesh-delivery-forge-bridge-tools] the prune of performed keeps its defaults (%d %%, to %d %%): %v\n", stdio.PruneMarkDefault, stdio.PruneTargetDefault, err) + p = stdio.Prune{Mark: stdio.PruneMarkDefault, Target: stdio.PruneTargetDefault} + } + return stdio.State(groups.PerformedState).Pruned(p) +} + // askCaller calls a tool as this bundle's module, through the runtime (stdio.Ask), the module's own invokes granting it. func askCaller(_ context.Context, key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) @@ -145,6 +168,7 @@ func main() { _, err := recorder.Record(meshhandactsclient.Request{What: what, Why: why, Cause: "owed-dropped"}) return err }, time.Now)...) + tools = append(tools, PerformedPruneTools(prunedPerformed(), recorder)...) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-forge-bridge-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go index 832f48ff5..276312f72 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go @@ -10,6 +10,7 @@ import ( seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-forge-bridge/internal/groups" ) @@ -164,4 +165,45 @@ func TestTheBridgeTools(t *testing.T) { t.Error("an ask naming no seat was taken") } }) + // Design 47 H2 §812 (H-A-380 (3), H-A-433): the bridge serves the two prune tools over its state performed, named for + // its module, each written in the hand-act log before it acts. + t.Run("the prune tools over performed", func(t *testing.T) { + p := &prunable{} + var recorded []views.HandAct + tools := map[string]stdio.Tool{} + for _, tool := range PerformedPruneTools(p, handActs(func(a views.HandAct) error { recorded = append(recorded, a); return nil })) { + tools[tool.Name] = tool + } + now, keep := tools["mesh-delivery-forge-bridge_state_prune_now"], tools["mesh-delivery-forge-bridge_state_keep"] + if len(tools) != 2 || now.Run == nil || keep.Run == nil { + t.Fatalf("the prune tools are %v, want mesh-delivery-forge-bridge_state_prune_now and _state_keep", tools) + } + until := time.Now().Add(time.Hour).UTC().Format(time.RFC3339) + if _, err := now.Run(map[string]any{"entry": "performed", "why": "full"}); err != nil || p.pruned != 1 { + t.Errorf("prune now: %v, pruned %d times", err, p.pruned) + } + if _, err := keep.Run(map[string]any{"entry": "performed", "key": "mesh-delivery.YQ", "until": until, "why": "read it"}); err != nil || p.kept != "mesh-delivery.YQ" { + t.Errorf("keep: %v, kept %q", err, p.kept) + } + if len(recorded) != 2 { + t.Errorf("hand acts %+v, want the prune and the keep", recorded) + } + }) } + +// prunable is a pruned state in memory. +type prunable struct { + pruned int + kept string +} + +func (p *prunable) PruneNow() (stdio.Pruned, error) { + p.pruned++ + return stdio.Pruned{State: "performed"}, nil +} +func (p *prunable) Keep(key string, _ time.Time) error { p.kept = key; return nil } + +// handActs records a hand act through record. +type handActs func(views.HandAct) error + +func (h handActs) RecordHandAct(a views.HandAct) error { return h(a) } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound.go index f92d1ca7d..63fc0b06c 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound.go @@ -10,6 +10,7 @@ import ( "github.com/nats-io/nats.go/jetstream" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -38,10 +39,15 @@ type boundBucket struct { binding *process.Binding local string js process.BucketOpener + // prune, when set, prunes the entry toward its cap at the bridge's own writes (design 47 H2 §812; H-A-380): the + // bucket is bound through mesh-sdk's process.PrunedState, its figures read once when it is opened (again for a + // recreated entry's new bucket) and after each prune, and every Put is counted toward prune-mark. + prune *stdio.Prune - mu sync.Mutex - name string - kv jetstream.KeyValue + mu sync.Mutex + name string + kv jetstream.KeyValue + pruned *process.PrunedState } func (s *boundBucket) open(ctx context.Context) (jetstream.KeyValue, error) { @@ -61,7 +67,13 @@ func (s *boundBucket) open(ctx context.Context) (jetstream.KeyValue, error) { if err != nil { return nil, fmt.Errorf("the bus entry %q, bucket %s: %w", s.local, names.Bucket, err) } - s.name, s.kv = names.Bucket, kv + var pruned *process.PrunedState + if s.prune != nil { + if pruned, err = process.StateOver(forgeBridgeModule(), s.local, kv).Pruned(*s.prune); err != nil { + return nil, fmt.Errorf("the bus entry %q, bucket %s, cannot be bound for its prune: %w", s.local, names.Bucket, err) + } + } + s.name, s.kv, s.pruned = names.Bucket, kv, pruned return kv, nil } @@ -73,11 +85,19 @@ func (s *boundBucket) Get(ctx context.Context, key string) (jetstream.KeyValueEn return kv.Get(ctx, key) } +// Put writes a key whole; for an entry bound for its prune the write is counted, and past prune-mark the entry is +// pruned to prune-target before Put returns (§812). func (s *boundBucket) Put(ctx context.Context, key string, value []byte) (uint64, error) { kv, err := s.open(ctx) if err != nil { return 0, err } + s.mu.Lock() + pruned := s.pruned + s.mu.Unlock() + if pruned != nil { + return pruned.Put(key, json.RawMessage(value)) + } return kv.Put(ctx, key, value) } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go index 260c87765..bbcdd7868 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go @@ -181,6 +181,9 @@ func answerOf(answer string) []byte { return []byte(`{"result":` + answer + `,"n // boundPerformed is the bus name the stand-in seat mesh-broker's show answers for the state performed. const boundPerformed = "mod_mesh-delivery-forge-bridge_performed_b1" +// performedCap is the cap of the bed's performed, small so a test fills it to its prune-mark (design 47 H2 §812). +const performedCap = 64 << 10 + // catalog is the repository the stand-in registrations name, its branch main watched (the seat mesh-catalog's fixture). const catalog = "novox/mesh-catalog" @@ -216,7 +219,11 @@ func startBed(t *testing.T, head string) *bed { } } for _, name := range []string{headsBucket(forgeBridgeModule()), boundPerformed, groupkeys.OwedAsksBucket(forgeBridgeModule())} { - if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 1}); err != nil { + var capped int64 + if name == boundPerformed { + capped = performedCap + } + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 1, MaxBytes: capped}); err != nil { t.Fatal(err) } } @@ -624,4 +631,102 @@ func TestBridgePerformsWhatIsOwedOnce(t *testing.T) { t.Errorf("performed of %s keeps %s, want the outcome, when and the pull request alone", g.ID, e.Value()) } refusedRead(t, b, "mesh-delivery-gate_owed") + + // Design 47 H2 §812 (F2, the bridge's whole test): a write that takes performed past prune-mark (70 % of its cap) + // prunes it, oldest acts first, down to prune-target (50 %); a key kept under _keep/ survives the prune. performed is + // filled while the bridge is stopped, so the bridge's bind at its start reads the fill once (H-A-380 (1)). + fillKey := func(i int) string { return performedKey(seat.Seat, fmt.Sprintf("fill-%03d", i)) } + fillValue := func(i, pad int) []byte { + v, _ := json.Marshal(Performed{Ended: forgemsg.OwedEnded{V: forgemsg.Version, Of: seat.Seat, ID: fmt.Sprintf("fill-%03d", i), + Outcome: forgemsg.OwedRefused, Words: strings.Repeat("w", pad), At: time.Now().UTC()}, Repository: "novox/mesh-sdk", Number: 1}) + return v + } + fillOf := func() uint64 { + t.Helper() + st, err := kept.Status(b.ctx) + if err != nil { + t.Fatal(err) + } + return st.Bytes() + } + mark, target := uint64(performedCap*70/100), uint64(performedCap*50/100) + fills, lastPut := 0, uint64(0) + b.restart(func() { + // The oldest filler (written below) kept from the prune for an hour, as mesh-delivery-forge-bridge_state_keep writes it. + keep, _ := json.Marshal(map[string]any{"until": time.Now().Add(time.Hour).UTC()}) + if _, err := kept.Put(b.ctx, "_keep/"+fillKey(0), keep); err != nil { + t.Fatal(err) + } + // Fill to 150 bytes below the mark: the last filler's words make up what is left, by the bytes one write was seen to put. + for { + before := fillOf() + if before+1200 >= mark { + per := uint64(600) // a put's bytes beside its value, as the first filler showed + if fills > 0 { + per = lastPut - uint64(len(fillValue(fills-1, 300))) + } + pad := int(mark - 150 - before - per - uint64(len(fillValue(fills, 0)))) + if _, err := kept.Put(b.ctx, fillKey(fills), fillValue(fills, pad)); err != nil { + t.Fatal(err) + } + fills++ + break + } + if _, err := kept.Put(b.ctx, fillKey(fills), fillValue(fills, 300)); err != nil { + t.Fatal(err) + } + lastPut = fillOf() - before + fills++ + } + }) + if f := fillOf(); f >= mark || f+400 < mark { + t.Fatalf("performed is filled to %d bytes, want just below its mark %d", f, mark) + } + pruned, err := kept.Watch(b.ctx, fillKey(1), jetstream.UpdatesOnly()) + if err != nil { + t.Fatal(err) + } + defer pruned.Stop() + other["id"] = "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/prune" + pruning, _ := json.Marshal(other) + if _, err := b.js.Publish(b.ctx, subject, pruning); err != nil { + t.Fatal(err) + } + if e := ended(); e.Outcome != forgemsg.OwedPerformed { + t.Errorf("the act that takes performed past its mark: %+v, want performed", e) + } + for gone, bound := false, time.After(20*time.Second); !gone; { + select { + case e := <-pruned.Updates(): + gone = e != nil && e.Operation() == jetstream.KeyValuePurge + case <-bound: + t.Fatalf("performed past its mark (%d of %d bytes) was not pruned within 20 s", fillOf(), uint64(performedCap)) + } + } + held := func(key string) bool { + _, err := kept.Get(b.ctx, key) + return err == nil + } + if !held(fillKey(0)) || !held("_keep/"+fillKey(0)) { + t.Errorf("the act kept under _keep/ was pruned") + } + if !held(performedKey(seat.Seat, "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/prune")) || !held(fillKey(fills-1)) { + t.Errorf("the newest acts were pruned") + } + // What the current values hold is down to the target: the bytes left, counted by mesh-sdk's upper bound per write (its + // key, its value and PutOverhead), which reads a little above the bus's own bytes the prune counted down from, so within + // 4 % of the cap; each purged key's marker aside. The bucket as the bus counts it is below the mark again. + keysLeft, err := kept.ListKeys(b.ctx) + if err != nil { + t.Fatal(err) + } + var left uint64 + for k := range keysLeft.Keys() { + if e, err := kept.Get(b.ctx, k); err == nil { + left += uint64(len(k) + len(e.Value()) + 96) + } + } + if left > target+performedCap*4/100 || left < target/2 || fillOf() >= mark { + t.Errorf("performed holds %d bytes of current values after its prune, want about its target %d (cap %d)", left, target, uint64(performedCap)) + } } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index 5c3b26e6f..9c6ed0e0f 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -16,6 +16,7 @@ import ( "syscall" "time" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/views" @@ -106,7 +107,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if w := binding.Waiting(); w != "" { logger.Printf("the bridge %s: nothing owed to the forge is performed until it is bound", w) } - performedKV := &boundBucket{binding: binding, local: performedLocal, js: js} + // performed is pruned toward its cap at the bridge's own writes (design 47 H2 §812): past prune-mark the oldest acts + // go down to prune-target, in one journal line, a key kept under _keep/ staying. A setting that is no percentage is + // said, and the defaults hold. + prune, err := stdio.PruneSettings() + if err != nil { + logger.Printf("the prune of performed keeps its defaults (%d %%, to %d %%): %v", stdio.PruneMarkDefault, stdio.PruneTargetDefault, err) + prune = stdio.Prune{Mark: stdio.PruneMarkDefault, Target: stdio.PruneTargetDefault} + } + performedKV := &boundBucket{binding: binding, local: performedLocal, js: js, prune: &prune} // What the gate owes the forge, performed through the seat git (design 47 *What the gate owes the forge*). owe := &owing{call: call, pub: js, performed: kvPerformed{performedKV}, acts: streamActs{js: js}, waits: binding.Waiting, now: time.Now, logger: logger} keptCall := keptCallOf(ctx, call) diff --git a/modules/mesh-delivery-forge-bridge/manifest_test.go b/modules/mesh-delivery-forge-bridge/manifest_test.go index 3c81cb8dd..90bd18e1e 100644 --- a/modules/mesh-delivery-forge-bridge/manifest_test.go +++ b/modules/mesh-delivery-forge-bridge/manifest_test.go @@ -26,6 +26,18 @@ type stream struct { type state struct { Name string `json:"name"` History int `json:"history"` + MaxMiB int `json:"max-mib"` +} + +// setting is an entry of config. +type setting struct { + Type string `json:"type"` + Description string `json:"description"` + Trusted bool `json:"trusted"` + Runtime bool `json:"runtime"` + Minimum float64 `json:"minimum"` + Maximum float64 `json:"maximum"` + Default any `json:"default"` } // kept is an entry of requires in its long form: kept data on the bus (novox/hq ADR 0595). @@ -49,19 +61,20 @@ type artifact struct { } type manifest struct { - Module string `json:"module"` - Version string `json:"version"` - Slug string `json:"slug"` - Consumes []string `json:"consumes"` - Streams []stream `json:"streams"` - State []state `json:"state"` - Requires []kept `json:"requires"` - Reads []string `json:"reads"` - Tools []string `json:"tools"` - Invokes []string `json:"invokes"` - OwnSecrets map[string]string `json:"own-secrets"` - SecretsOwner string `json:"secrets-owner"` - Resources []map[string]any `json:"resources"` + Module string `json:"module"` + Version string `json:"version"` + Slug string `json:"slug"` + Consumes []string `json:"consumes"` + Streams []stream `json:"streams"` + State []state `json:"state"` + Requires []kept `json:"requires"` + Reads []string `json:"reads"` + Tools []string `json:"tools"` + Invokes []string `json:"invokes"` + Config map[string]setting `json:"config"` + OwnSecrets map[string]string `json:"own-secrets"` + SecretsOwner string `json:"secrets-owner"` + Resources []map[string]any `json:"resources"` Build struct { Artifacts []artifact `json:"artifacts"` } `json:"build"` @@ -125,20 +138,28 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 if m.SecretsOwner != "mesh-delivery-forge-bridge" { t.Errorf("secrets-owner = %q", m.SecretsOwner) } - // The head held for each watched branch (design 47 H §739), and the asks about what is owed. What it performed of what is - // owed is kept data in the new form, one key per act, kept 7 days as the owner's stream owed is; its size is the - // template's own (design 47 *What the gate owes the forge*, ADR 0595 §3). No inputs, end-asks or findings (H §764, H-A-48). - if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "owed-asks", History: 1}}) { + // The head held for each watched branch (design 47 H §739), and the asks about what is owed, each capped at 8 MiB + // (design 47 H2 §812, H-A-411 (2)). What it performed of what is owed is kept data in the new form, one key per act, + // kept 3 days by the bucket's TTL, performed-keep, and capped at 8 MiB, pruned toward it at the bridge's writes (§812, + // H-A-433). No inputs, end-asks or findings (H §764, H-A-48). + if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1, MaxMiB: 8}, {Name: "owed-asks", History: 1, MaxMiB: 8}}) { t.Errorf("state = %+v, want the buckets heads and owed-asks", m.State) } - if want := []kept{{Name: "performed", Storage: "key-value", Template: "state", Options: map[string]any{"history": float64(1), "ttl-seconds": float64(604800)}}}; !reflect.DeepEqual(m.Requires, want) { + if want := []kept{{Name: "performed", Storage: "key-value", Template: "state", Options: map[string]any{"history": float64(1), "ttl-seconds": float64(259200), "max-mib": float64(8)}}}; !reflect.DeepEqual(m.Requires, want) { t.Errorf("requires = %+v, want performed alone as %+v", m.Requires, want) } // bridge_tags reads a package's tags for the builder's floor (design 51); bridge_heads reads the state heads (design 47 - // H-A-85); the owed tools ask the bridge through owed-asks. - if want := []string{"bridge_heads", "bridge_owed", "bridge_owed_again", "bridge_owed_drop", "bridge_tags"}; !reflect.DeepEqual(sorted(m.Tools), want) { + // H-A-85); the owed tools ask the bridge through owed-asks; the two prune tools act on performed (§812, H-A-380 (3)). + if want := []string{"bridge_heads", "bridge_owed", "bridge_owed_again", "bridge_owed_drop", "bridge_tags", + "mesh-delivery-forge-bridge_state_keep", "mesh-delivery-forge-bridge_state_prune_now"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } + // performed's prune starts past prune-mark and ends at prune-target, settings read at run time (§812). + for key, def := range map[string]float64{"prune-mark": 70, "prune-target": 50} { + if c := m.Config[key]; c.Default != any(def) || !c.Runtime { + t.Errorf("config %s = %+v, want a runtime setting defaulting to %v", key, c, def) + } + } // Design 47 H §764 (B3 AC1): the earlier model's names are in no file of the bridge but tests asserting their absence. retired := []string{"pushed", "forge.group.merged", "bridge_groups", "bridge_group_end_again", "bridge_merged_message", "OwedActMerge", "mesh-delivery-checker", "seat:git.merge-pull", "branch-unprotected", "ForgeMerged", "ForgePushed"} diff --git a/modules/mesh-delivery-forge-bridge/module.json b/modules/mesh-delivery-forge-bridge/module.json index 2cdf59b7a..b5a412112 100644 --- a/modules/mesh-delivery-forge-bridge/module.json +++ b/modules/mesh-delivery-forge-bridge/module.json @@ -14,15 +14,17 @@ { "name": "forge", "events": ["forge.**", "owed.ended"], "max-age-days": 7 } ], "state": [ - { "name": "heads", "history": 1 }, - { "name": "owed-asks", "history": 1 } + { "name": "heads", "history": 1, "max-mib": 8 }, + { "name": "owed-asks", "history": 1, "max-mib": 8 } ], "tools": [ "bridge_tags", "bridge_heads", "bridge_owed", "bridge_owed_again", - "bridge_owed_drop" + "bridge_owed_drop", + "mesh-delivery-forge-bridge_state_prune_now", + "mesh-delivery-forge-bridge_state_keep" ], "invokes": [ "seat:git.status", @@ -41,9 +43,29 @@ "name": "performed", "storage": "key-value", "template": "state", - "options": { "history": 1, "ttl-seconds": 604800 } + "options": { "history": 1, "ttl-seconds": 259200, "max-mib": 8 } } ], + "config": { + "prune-mark": { + "type": "number", + "description": "The fill of the bridge's state performed, in percent of its cap, past which the bridge prunes it at its own write, oldest acts first (design 47 H2 §812): below the bus module's raise mark of 75 %.", + "trusted": false, + "runtime": true, + "minimum": 1, + "maximum": 100, + "default": 70 + }, + "prune-target": { + "type": "number", + "description": "The fill, in percent of the cap, a prune of the bridge's state performed takes it down to (design 47 H2 §812); below prune-mark.", + "trusted": false, + "runtime": true, + "minimum": 1, + "maximum": 100, + "default": 50 + } + }, "own-secrets": { "broker": "${dir:state}/broker" }, From 13d960a12078b531ca5924e9df45e79ea61ac5ef Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:46:49 +0100 Subject: [PATCH 11/12] F1: an owed act failing on a part away stays owed, and a record's acts are performed again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An owed act met by the forge away (401 while the store was replaced) ended could not be performed and nothing performed it again. Each act is now a kept call at the module's pace (kept-call-pace-first, kept-call-pace-max) until owed-away-bound (60 minutes), then ends could not be performed naming every try. Before each try the bridge reads the pull request through the seat git's pull: one closed unmerged ends owed.ended not-needed, never performed. The gate's owed event gains record (the delivery record's source revision), kept with the act in performed, and bridge_owed_again by record and why performs again every act of that record that ended could not be performed, written in the hand-act log first. Builds: design 47 H2 §807, §775, §819, H-A-432 --- .../go/seats/meshdelivery/contract.go | 4 +- .../seats/mesh-delivery/seat.json | 4 + .../ts/seats/meshdelivery/index.d.ts | 2 + .../mesh-delivery-forge-bridge-tools/main.go | 4 +- .../main_test.go | 19 +- .../mesh-delivery-forge-bridge-tools/owed.go | 44 +++- .../mesh-delivery-forge-bridge/bound_test.go | 2 +- .../mesh-delivery-forge-bridge/gate_test.go | 83 +++++++- .../integration_test.go | 55 +++++ .../cmd/mesh-delivery-forge-bridge/main.go | 26 ++- .../mesh-delivery-forge-bridge/main_test.go | 1 + .../cmd/mesh-delivery-forge-bridge/owed.go | 190 +++++++++++++++--- .../forgemsg/forge_test.go | 3 + .../forgemsg/owed.go | 8 +- .../internal/bridge/owed.go | 9 +- .../internal/bridge/owed_test.go | 7 + .../internal/groups/owedask.go | 23 ++- .../internal/groups/performed.go | 7 +- .../manifest_test.go | 14 +- .../mesh-delivery-forge-bridge/module.json | 22 ++ 20 files changed, 463 insertions(+), 64 deletions(-) diff --git a/modules/mesh-delivery-contracts/go/seats/meshdelivery/contract.go b/modules/mesh-delivery-contracts/go/seats/meshdelivery/contract.go index 89487ec9b..17f60d581 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshdelivery/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshdelivery/contract.go @@ -20,7 +20,7 @@ const Form = 2 // whose fields changed while its forms did not (design 47 H2 §780). const ( generatedForms = "2" - generatedFieldsDigest = "DeliveriesAnswer=d6ae3ca4e5da Fill=80f442916429 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 HeadsByRepositoryBranchValue=b32fd594ee25 Owed=005330be954f OwedByRepositoryNumberHeadActValue=f16ff5733d62 OwedEvent=86af3b56de28 RecordEndedEvent=566edb8684e5 RecordEndedEventAwayItem=44ad7d26bd16 RecordOpenedEvent=9cf9afaa25b4 RecoveryAnswer=8c705e17cdf8 RecoveryMergedEvent=014d4d00d2e5 RoomEvent=5bd4df78ed84 ShowAnswer=7538f2f5e68e SourceAnswer=6ae84f05f10d SourceByBranchValue=31aa822718c0 SourceHead=e722e2d9492f SourceMovedEvent=5231909928e9 SourceMovedEventHeadsItem=774b4bf88ba4 StalledAnswer=7538f2f5e68e" + generatedFieldsDigest = "DeliveriesAnswer=d6ae3ca4e5da Fill=80f442916429 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 HeadsByRepositoryBranchValue=b32fd594ee25 Owed=a30ee01924cc OwedByRepositoryNumberHeadActValue=f16ff5733d62 OwedEvent=86af3b56de28 RecordEndedEvent=566edb8684e5 RecordEndedEventAwayItem=44ad7d26bd16 RecordOpenedEvent=9cf9afaa25b4 RecoveryAnswer=8c705e17cdf8 RecoveryMergedEvent=014d4d00d2e5 RoomEvent=5bd4df78ed84 ShowAnswer=7538f2f5e68e SourceAnswer=6ae84f05f10d SourceByBranchValue=31aa822718c0 SourceHead=e722e2d9492f SourceMovedEvent=5231909928e9 SourceMovedEventHeadsItem=774b4bf88ba4 StalledAnswer=7538f2f5e68e" ) const ( @@ -244,6 +244,8 @@ type Owed struct { Line string `json:"line,omitempty"` // the pull request's number Number int64 `json:"number"` + // the delivery record the act is for, by its source revision, .; absent on an act said before it was named, which bridge_owed_again by record never finds (design 47 H2 §807, H-A-432) + Record string `json:"record,omitempty"` // the repository, / Repository string `json:"repository"` // when the gate first said it diff --git a/modules/mesh-delivery-contracts/seats/mesh-delivery/seat.json b/modules/mesh-delivery-contracts/seats/mesh-delivery/seat.json index eb8d844e3..08e74ab4e 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-delivery/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-delivery/seat.json @@ -649,6 +649,10 @@ "type": "string", "format": "date-time", "description": "when the gate first said it" + }, + "record": { + "type": "string", + "description": "the delivery record the act is for, by its source revision, .; absent on an act said before it was named, which bridge_owed_again by record never finds (design 47 H2 §807, H-A-432)" } }, "required": [ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshdelivery/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshdelivery/index.d.ts index ee0549577..f07a02274 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshdelivery/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshdelivery/index.d.ts @@ -110,6 +110,8 @@ export interface Owed { "line"?: string; /** the pull request's number */ "number": number; + /** the delivery record the act is for, by its source revision, .; absent on an act said before it was named, which bridge_owed_again by record never finds (design 47 H2 §807, H-A-432) */ + "record"?: string; /** the repository, / */ "repository": string; /** when the gate first said it */ diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go index a32745934..84d5bbdb6 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go @@ -164,8 +164,8 @@ func main() { tools = append(tools, OwedTools(askState(owedAsks, groups.OwedAsksBucket(served), 25*time.Second, func(v []byte) bool { a, err := groups.DecodeOwedAsk(v) return err == nil && a.Done() - }), func(f OwedFilter) ([]OwedRow, error) { return performedRows(performed, f) }, func(what, why string) error { - _, err := recorder.Record(meshhandactsclient.Request{What: what, Why: why, Cause: "owed-dropped"}) + }), func(f OwedFilter) ([]OwedRow, error) { return performedRows(performed, f) }, func(what, why, cause string) error { + _, err := recorder.Record(meshhandactsclient.Request{What: what, Why: why, Cause: cause}) return err }, time.Now)...) tools = append(tools, PerformedPruneTools(prunedPerformed(), recorder)...) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go index 276312f72..8291cf46a 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "reflect" + "strings" "testing" "time" @@ -111,7 +112,10 @@ func TestTheBridgeTools(t *testing.T) { } var recorded []string recordErr := error(nil) - record := func(what, why string) error { recorded = append(recorded, what+": "+why); return recordErr } + record := func(what, why, cause string) error { + recorded = append(recorded, what+": "+why+" ("+cause+")") + return recordErr + } tools := map[string]func(map[string]any) (any, error){} for _, tl := range OwedTools(ask, performed, record, func() time.Time { return at }) { tools[tl.Name] = tl.Run @@ -164,6 +168,19 @@ func TestTheBridgeTools(t *testing.T) { if _, _, err := groups.NewOwedAsk(groups.OwedAgain, "", "a", "", "", at); err == nil { t.Error("an ask naming no seat was taken") } + // Design 47 H2 §807 (H-A-432 (1)): bridge_owed_again by record asks the bridge for every act of one delivery record, + // written in the hand-act log first; with no why, or with an act named beside it, nothing is asked. + n, r := len(asked), len(recorded) + for _, bad := range []map[string]any{{"record": "main.212"}, {"record": "main.212", "of": seat.Seat, "id": "a", "why": "x"}} { + if _, err := tools[OwedAgainTool](bad); err == nil || len(asked) != n || len(recorded) != r { + t.Errorf("bridge_owed_again %v: %v, asked %d; want refused, nothing asked", bad, err, len(asked)-n) + } + } + if out, err := tools[OwedAgainTool](map[string]any{"record": "main.212", "why": "the forge was away"}); err != nil || + len(asked) != n+1 || asked[n].Record != "main.212" || asked[n].Kind != groups.OwedAgain || asked[n].ID != "" || + len(recorded) != r+1 || !strings.HasSuffix(recorded[r], "(owed-again)") { + t.Errorf("bridge_owed_again by record: %v %v, asked %+v, recorded %v", out, err, asked[n:], recorded[r:]) + } }) // Design 47 H2 §812 (H-A-380 (3), H-A-433): the bridge serves the two prune tools over its state performed, named for // its module, each written in the hand-act log before it acts. diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/owed.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/owed.go index 646861c44..73457f2a0 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/owed.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/owed.go @@ -72,9 +72,9 @@ func performedRows(s stdio.KeptState, f OwedFilter) ([]OwedRow, error) { } // OwedTools are bridge_owed, bridge_owed_again and bridge_owed_drop: ask writes an owed-asks value and waits for the -// bridge's answer; performed reads the state performed; record writes a hand act in the seat mesh-hand-acts' log before -// a drop is asked, so a drop is never made without its record. -func OwedTools(ask func(key string, value []byte) ([]byte, error), performed func(OwedFilter) ([]OwedRow, error), record func(what, why string) error, now func() time.Time) []stdio.Tool { +// bridge's answer; performed reads the state performed; record writes a hand act in the seat mesh-hand-acts' log (what, +// why and its cause) before a drop or a record's acts performed again is asked, so neither is made without its record. +func OwedTools(ask func(key string, value []byte) ([]byte, error), performed func(OwedFilter) ([]OwedRow, error), record func(what, why, cause string) error, now func() time.Time) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } act := map[string]any{ "of": map[string]any{"type": "string", "description": "the seat that said the act: mesh-delivery (the gate)"}, @@ -96,6 +96,12 @@ func OwedTools(ask func(key string, value []byte) ([]byte, error), performed fun for k, v := range act { read[k] = v } + againArgs := map[string]any{ + "record": map[string]any{"type": "string", "description": "instead of of and id: a delivery record's source revision, .; with why"}, + "why": map[string]any{"type": "string", "description": "with record: why its acts are performed again, in plain words"}} + for k, v := range act { + againArgs[k] = v + } withWhy := map[string]any{"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}} for k, v := range act { withWhy[k] = v @@ -104,7 +110,7 @@ func OwedTools(ask func(key string, value []byte) ([]byte, error), performed fun {Name: OwedTool, Description: "What the forge bridge performed of what the gate and the checker owe the forge (design 47): each act it said as " + "owed.ended, newest first, with its seat, id, pull request, outcome (performed, refused, could not be performed, " + - "undecodable, dropped), the forge's words and when, from its state performed (kept 7 days), of one act (id, of " + + "undecodable, dropped, not-needed), the forge's words and when, from its state performed (kept 3 days), of one act (id, of " + "narrowing it) or of one pull request (repository and number), never the whole state; and the backlog of its " + "durable on each seat's stream owed: the last event applied, the events waiting and held, the age of the oldest. " + "Read-only.", @@ -133,9 +139,33 @@ func OwedTools(ask func(key string, value []byte) ([]byte, error), performed fun Description: "Perform one act owed to the forge again now, by hand, whatever its last outcome (a refusal the forge no longer " + "makes, an act that could not be performed): the bridge calls the seat git's verb of the act under its id (safe to " + "repeat) and says its outcome as owed.ended. The act is read from the stream owed of the seat that said it, on the " + - "subject of the pull request the bridge's state performed names (kept 7 days there). Answers the outcome.", - Input: map[string]any{"type": "object", "required": []string{"of", "id"}, "properties": act}, + "subject of the pull request the bridge's state performed names (kept 3 days there). Or, with record (a delivery " + + "record's source revision, .) and why instead of of and id, every act of that record that ended " + + "could not be performed, written first in the hand-act log (design 47 H2 §807). Each act reads its pull request " + + "again before it acts: one closed unmerged ends not-needed. Answers the outcome, or each act's.", + Input: map[string]any{"type": "object", "properties": againArgs}, Run: func(args map[string]any) (any, error) { + if rec := str(args, "record"); rec != "" { + if str(args, "of") != "" || str(args, "id") != "" { + return nil, fmt.Errorf("bridge_owed_again names one act (of and id) or one record, not both") + } + key, value, err := groups.NewOwedRecordAsk(rec, str(args, "why"), stdio.CallerOf(args), now()) + if err != nil { + return nil, err + } + if err := record("performed again every act of the record "+rec+" that could not be performed", str(args, "why"), "owed-again"); err != nil { + return nil, err + } + v, err := ask(key, value) + if err != nil { + return nil, err + } + a, err := groups.DecodeOwedAsk(v) + if err != nil { + return nil, err + } + return map[string]any{"record": a.Record, "answer": a.Answer}, nil + } a, err := asked(groups.OwedAgain, args) if err != nil { return nil, err @@ -151,7 +181,7 @@ func OwedTools(ask func(key string, value []byte) ([]byte, error), performed fun if str(args, "why") == "" { return nil, fmt.Errorf("a drop says why") } - if err := record(fmt.Sprintf("dropped the act %s that %s owes the forge", str(args, "id"), str(args, "of")), str(args, "why")); err != nil { + if err := record(fmt.Sprintf("dropped the act %s that %s owes the forge", str(args, "id"), str(args, "of")), str(args, "why"), "owed-dropped"); err != nil { return nil, err } a, err := asked(groups.OwedDrop, args) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound_test.go index d521d33a9..480d8a757 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/bound_test.go @@ -48,7 +48,7 @@ func performedIsOpenedByItsBinding(t *testing.T) { o, pub, kept := newOwing(f) o.waits = binding.Waiting gate := gitfixture.Seat(t, seat.Seat, "events", "owed is said, a status", "event:owed")[0] - if _, err := o.heard(context.Background(), seat.Seat, gate); err == nil || len(f.calls) != 0 || len(pub.data) != 0 || len(kept) != 0 { + if _, err := o.heard(context.Background(), seat.Seat, gate, nil); err == nil || len(f.calls) != 0 || len(pub.data) != 0 || len(kept) != 0 { t.Errorf("unbound: %v, called %v, said %d; want the act left unperformed and unsaid", err, f.calls, len(pub.data)) } answered = true diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go index 517a56d25..b324f7b95 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "log" + "strconv" "strings" "testing" "time" @@ -14,6 +15,8 @@ import ( seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + "git.novox.be/novox/mesh-sdk/go/busretry" + "mesh-delivery-forge-bridge/internal/gitfixture" "mesh-delivery-forge-bridge/internal/groups" ) @@ -38,6 +41,13 @@ func (m memPerformed) Get(_ context.Context, key string) ([]byte, bool, error) { return v, ok, nil } func (m memPerformed) Put(_ context.Context, key string, v []byte) error { m[key] = v; return nil } +func (m memPerformed) Keys(context.Context) ([]string, error) { + var out []string + for k := range m { + out = append(out, k) + } + return out, nil +} // memActs is a seat's stream owed by pull request subject (`#`); asked records what was read. type memActs struct { @@ -58,13 +68,28 @@ func (m *memActs) act(_ context.Context, of, repository string, number int, id s } // seatFake answers each seat verb from answers, by its key; one absent is not answered (an error, as a bound passed). +// The seat git's pull, read before each try of an act (H-A-432), is counted apart in pulls and answers an open pull +// request unless answers names it; away answers the act verbs `away` that many times first (§807). type seatFake struct { calls []string + pulls int + away int answers map[string]json.RawMessage } func (f *seatFake) call(_ context.Context, key string, _ any) (json.RawMessage, error) { + if key == "seat:git.pull" { + f.pulls++ + if a, ok := f.answers[key]; ok { + return a, nil + } + return json.RawMessage(`{"pull":{"repository":"novox/mesh-sdk","number":178,"branch":"b","head":"` + headA + `","base":"main","state":"open"}}`), nil + } f.calls = append(f.calls, key) + if f.away > 0 { + f.away-- + return json.Marshal(busretry.AwayAnswer("the forge", "401 while the store is replaced")) + } if a, ok := f.answers[key]; ok { return a, nil } @@ -104,11 +129,16 @@ func TestWhatIsOwedIsPerformedSaidAndKept(t *testing.T) { {"a seat that says no owed act", "mesh-wants", gate("owed is said, a status"), nil, nil, "", forgemsg.OwedUndecodable, false, ""}, {"the stream forge refusing the outcome", seat.Seat, gate("owed is said, a status"), statusOK, errors.New("no responders"), "seat:git.status", "", false, "could not be said"}, + // Design 47 H2 §807 (H-A-432 (2)): a pull request closed unmerged, read before the try, ends the act not-needed. + {"a status whose pull request closed unmerged", seat.Seat, gate("owed is said, a status"), + map[string]json.RawMessage{"seat:git.status": answer("status", "status answers"), + "seat:git.pull": json.RawMessage(`{"pull":{"repository":"novox/mesh-sdk","number":178,"branch":"b","head":"` + headA + `","base":"main","state":"closed"}}`)}, + nil, "", forgemsg.OwedNotNeeded, true, ""}, } { f := &seatFake{answers: c.answers} o, pub, kept := newOwing(f) pub.err = c.pubErr - _, err := o.heard(context.Background(), c.of, c.data) + _, err := o.heard(context.Background(), c.of, c.data, nil) if c.err != "" { if err == nil || !strings.Contains(err.Error(), c.err) || len(kept) != 0 { t.Errorf("%s: %v, kept %d; want an error naming %q and nothing kept", c.name, err, len(kept), c.err) @@ -138,6 +168,55 @@ func TestWhatIsOwedIsPerformedSaidAndKept(t *testing.T) { } } + // Design 47 H2 §807, §775: an act met by the seat git's away is called again at the kept-call pace, its pull request + // read before each try, and performed when the forge answers; one away past owed-away-bound ends could not be + // performed, naming every try. + t.Run("an act through a part away", func(t *testing.T) { + defer busretry.Waits(time.Minute, time.Millisecond, 2*time.Millisecond)() + f := &seatFake{answers: statusOK, away: 2} + o, pub, _ := newOwing(f) + if e, err := o.heard(context.Background(), seat.Seat, gate("owed is said, a status"), nil); err != nil || e.Outcome != forgemsg.OwedPerformed || + len(f.calls) != 3 || f.pulls != 3 || len(pub.data) != 1 { + t.Errorf("away twice: %+v %v, called %v, pulls %d; want performed at the third try, its pull request read before each", e, err, f.calls, f.pulls) + } + f = &seatFake{answers: statusOK, away: 1 << 30} + o, _, _ = newOwing(f) + o.awayBound = 10 * time.Millisecond + if e, err := o.heard(context.Background(), seat.Seat, gate("owed is said, a status"), nil); err != nil || e.Outcome != forgemsg.OwedCouldNotBePerformed || + !strings.Contains(e.Words, "called "+strconv.Itoa(len(f.calls))+" times") || len(f.calls) < 2 { + t.Errorf("away past the bound: %+v %v after %d calls; want could not be performed naming every try", e, err, len(f.calls)) + } + }) + + // Design 47 H2 §807 (H-A-432 (1)): bridge_owed_again by record performs again every act of that record that ended + // could not be performed, found in performed by the record kept with it; an act of it that ended otherwise, and one + // of another record, are left. + t.Run("a record's acts performed again", func(t *testing.T) { + f := &seatFake{answers: statusOK} + o, pub, kept := newOwing(f) + status := "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/merge-gate" + o.acts = &memActs{held: map[string][][]byte{"mesh-delivery novox/mesh-sdk#178": {gate("owed is said, a status")}}} + at := time.Date(2026, 11, 9, 20, 0, 0, 0, time.UTC) + for id, c := range map[string]struct{ record, outcome string }{ + status: {"main.212", forgemsg.OwedCouldNotBePerformed}, "done": {"main.212", forgemsg.OwedPerformed}, + "another": {"main.213", forgemsg.OwedCouldNotBePerformed}} { + kept[performedKey(seat.Seat, id)], _ = json.Marshal(Performed{Ended: forgemsg.OwedEnded{V: 1, Of: seat.Seat, ID: id, + Outcome: c.outcome, Words: "x", At: at}, Repository: "novox/mesh-sdk", Number: 178, Record: c.record}) + } + a := o.answer(context.Background(), groups.OwedAsk{Kind: groups.OwedAgain, Record: "main.212", Why: "the forge was away"}, nil) + if !a.Done() || !strings.Contains(a.Answer, "1 acts of the record main.212") || !strings.Contains(a.Answer, status+": performed") || + len(f.calls) != 1 || f.pulls != 1 || len(pub.data) != 1 { + t.Errorf("again by record: %+v, called %v, pulls %d, said %d; want the one act that could not be performed, performed", a, f.calls, f.pulls, len(pub.data)) + } + var p Performed + if err := json.Unmarshal(kept[performedKey(seat.Seat, status)], &p); err != nil || p.Ended.Outcome != forgemsg.OwedPerformed || p.Record != "main.212" { + t.Errorf("performed after again by record: %+v %v", p, err) + } + if a := o.answer(context.Background(), groups.OwedAsk{Kind: groups.OwedAgain, Record: "main.212"}, nil); !strings.Contains(a.Answer, "holds no act") { + t.Errorf("again by record, nothing left: %+v", a) + } + }) + t.Run("heard from a seat's stream owed", func(t *testing.T) { f := &seatFake{answers: statusOK} o, pub, _ := newOwing(f) @@ -168,7 +247,7 @@ func TestWhatIsOwedIsPerformedSaidAndKept(t *testing.T) { acts := &memActs{held: map[string][][]byte{"mesh-delivery novox/mesh-sdk#178": {[]byte(`{`), gate("owed is said, a status")}}} o.acts = acts status := "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/merge-gate" - if _, err := o.heard(context.Background(), seat.Seat, gate("owed is said, a status")); err != nil { + if _, err := o.heard(context.Background(), seat.Seat, gate("owed is said, a status"), nil); err != nil { t.Fatal(err) } backlog := func(context.Context) []groups.Backlog { return []groups.Backlog{{Stream: seat.StreamOwed, Waiting: 2}} } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go index bbcdd7868..fb1e081fa 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go @@ -22,6 +22,8 @@ import ( git "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" @@ -111,6 +113,8 @@ accounts { "mesh.seat.git.tool.note" "mesh.seat.git.call.tags.n1~mesh-delivery-forge-bridge" "mesh.seat.git.tool.tags" + "mesh.seat.git.call.pull.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.pull" "mesh.seat.mesh-hand-acts.call.hand-act.n1~mesh-delivery-forge-bridge" "mesh.seat.mesh-hand-acts.tool.hand-act" "mesh.stream.mesh-delivery-forge-bridge.event.forge.>" @@ -178,6 +182,11 @@ type bed struct { // answerOf is a verb's answer in the reply envelope of mesh-sdk go/process/call.go ({result, error, node}). func answerOf(answer string) []byte { return []byte(`{"result":` + answer + `,"node":"n0"}`) } +// pullAnswer is the seat git's pull answering one pull request in state (open, merged or closed). +func pullAnswer(repository string, number int, state string) string { + return fmt.Sprintf(`{"pull":{"repository":%q,"number":%d,"branch":"b","head":"%s","base":"main","state":%q}}`, repository, number, strings.Repeat("a", 40), state) +} + // boundPerformed is the bus name the stand-in seat mesh-broker's show answers for the state performed. const boundPerformed = "mod_mesh-delivery-forge-bridge_performed_b1" @@ -248,6 +257,7 @@ func startBed(t *testing.T, head string) *bed { "mesh-delivery.source": `{"branches":["main"]}`, "mesh-broker.show": `{"state":"created","binding":{"local":"performed","storage":"key-value","template":"state","bucket":"` + boundPerformed + `"}}`, "git.branch " + catalog: `{"repository":"` + catalog + `","branch":"main","head":"` + head + `","protected":true}`, + "git.pull": pullAnswer("novox/mesh-sdk", 178, "open"), } for _, verb := range []string{"status", "comment", "view", "note"} { answers["git."+verb] = string(gitfixture.Given(t, verb+" answers", "verb:"+verb+".answer")) @@ -488,6 +498,10 @@ func refusedRead(t *testing.T, b *bed, bucket string) { // Acceptance 49: the forge bridge sets the group on each member's merged message and says the group's end once, after the // last member's, from the seat mesh-delivery's verb groups; its account reads no state of the gate's. func TestBridgePerformsWhatIsOwedOnce(t *testing.T) { + // The bridge's kept-call pace, read at its start (§775), in milliseconds here so the forge's minute away is short. + t.Cleanup(busretry.Waits(busretry.Bound, busretry.FirstWait, busretry.MostWait)) + t.Setenv(stdio.SettingWord(busretry.PaceFirstSetting), "10ms") + t.Setenv(stdio.SettingWord(busretry.PaceMaxSetting), "40ms") b := startBed(t, strings.Repeat("1", 40)) ch := heard(t, b) ended := func() forgemsg.OwedEnded { @@ -632,6 +646,47 @@ func TestBridgePerformsWhatIsOwedOnce(t *testing.T) { } refusedRead(t, b, "mesh-delivery-gate_owed") + // Design 47 H2 §807 (F1, the bridge's whole test): with the forge away (the seat git answering away to every call), a + // note line owed in that time is performed once the forge answers again, with no hand act; an act whose pull request + // closed unmerged meanwhile ends not-needed, never performed (H-A-432 (2), (3)). + away, _ := json.Marshal(busretry.AwayAnswer("the forge", "401 while the store is replaced")) + for _, verb := range []string{"git.pull", "git.note", "git.comment"} { + b.answer(verb, string(away)) + } + handActs, pulls, comments := callsOf(b, "mesh-hand-acts.hand-act"), callsOf(b, "git.pull"), callsOf(b, "git.comment") + note := map[string]any{"id": "note.novox/mesh-sdk#178.aaaaaaaaaaaa.away", "act": "note", "repository": "novox/mesh-sdk", + "number": 178, "head": strings.Repeat("a", 40), "line": "built while the forge was away"} + comment := map[string]any{"id": "comment.novox/mesh-catalog#9.aaaaaaaaaaaa.away", "act": "comment", "repository": catalog, + "number": 9, "head": strings.Repeat("a", 40), "text": "closed meanwhile"} + for _, act := range []map[string]any{note, comment} { + data, _ := json.Marshal(act) + if _, err := b.js.Publish(b.ctx, seat.SubjectOwed(keys.EscapeName(act["repository"].(string)), fmt.Sprint(act["number"])), data); err != nil { + t.Fatal(err) + } + } + calledTimes(t, b, "git.pull", pulls+3) // tried, met away, and tried again at the pace + b.answer("git.pull "+"novox/mesh-sdk", pullAnswer("novox/mesh-sdk", 178, "merged")) + b.answer("git.pull "+catalog, pullAnswer(catalog, 9, "closed")) + b.answer("git.note", string(gitfixture.Given(t, "note answers", "verb:note.answer"))) + // Read until both acts ended (bridge_owed_again's outcome above is on the stream before them). + outcomes := map[string]forgemsg.OwedEnded{} + for i := 0; i < 4 && (outcomes[note["id"].(string)].ID == "" || outcomes[comment["id"].(string)].ID == ""); i++ { + e := ended() + outcomes[e.ID] = e + } + if e := outcomes[note["id"].(string)]; e.Outcome != forgemsg.OwedPerformed { + t.Errorf("the note owed while the forge was away: %+v, want performed once it answers", e) + } + if e := outcomes[comment["id"].(string)]; e.Outcome != forgemsg.OwedNotNeeded || !strings.Contains(e.Words, "closed unmerged") { + t.Errorf("the comment on a pull request closed meanwhile: %+v, want not-needed naming it", e) + } + if n := callsOf(b, "git.comment"); n != comments { + t.Errorf("the comment on a pull request closed meanwhile was asked of the forge %d times, want never", n-comments) + } + if n := callsOf(b, "mesh-hand-acts.hand-act"); n != handActs { + t.Errorf("%d hand acts while the forge was away, want none", n-handActs) + } + // Design 47 H2 §812 (F2, the bridge's whole test): a write that takes performed past prune-mark (70 % of its cap) // prunes it, oldest acts first, down to prune-target (50 %); a key kept under _keep/ survives the prune. performed is // filled while the bridge is stopped, so the bridge's bind at its start reads the fill once (H-A-380 (1)). diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index 9c6ed0e0f..652911956 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -17,6 +17,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/views" @@ -58,10 +59,18 @@ type delivered interface { Data() []byte Headers() nats.Header Ack() error + InProgress() error Term() error NakWithDelay(delay time.Duration) error } +// owedAwayBoundSetting is the setting owed-away-bound: how long an owed act is called again through a part away before +// it ends could not be performed (design 47 H2 §807), 60 minutes when none is composed. +const ( + owedAwayBoundSetting = "owed-away-bound" + owedAwayBoundDefault = 60 * time.Minute +) + // retryDelay is how long the server waits before it delivers again an event the forge could not be asked about (the // server's redelivery is the only clock; issue 1236/AC3). const retryDelay = 10 * time.Second @@ -107,6 +116,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if w := binding.Waiting(); w != "" { logger.Printf("the bridge %s: nothing owed to the forge is performed until it is bound", w) } + // Each owed act is a kept call (design 47 H2 §807, §775): at the module's kept-call pace, until owed-away-bound. A + // setting that is no duration is said, and its default holds. + if err := busretry.ReadPace(stdio.SettingDuration); err != nil { + logger.Printf("the kept-call pace keeps %s doubling to %s: %v", busretry.KeptPace.First, busretry.KeptPace.Max, err) + } + awayBound, err := stdio.SettingDuration(owedAwayBoundSetting, owedAwayBoundDefault) + if err != nil { + logger.Printf("owed-away-bound keeps its default %s: %v", owedAwayBoundDefault, err) + awayBound = owedAwayBoundDefault + } // performed is pruned toward its cap at the bridge's own writes (design 47 H2 §812): past prune-mark the oldest acts // go down to prune-target, in one journal line, a key kept under _keep/ staying. A setting that is no percentage is // said, and the defaults hold. @@ -117,7 +136,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } performedKV := &boundBucket{binding: binding, local: performedLocal, js: js, prune: &prune} // What the gate owes the forge, performed through the seat git (design 47 *What the gate owes the forge*). - owe := &owing{call: call, pub: js, performed: kvPerformed{performedKV}, acts: streamActs{js: js}, waits: binding.Waiting, now: time.Now, logger: logger} + owe := &owing{call: call, pub: js, performed: kvPerformed{performedKV}, acts: streamActs{js: js}, waits: binding.Waiting, + awayBound: awayBound, now: time.Now, logger: logger} keptCall := keptCallOf(ctx, call) reader := &views.ProcessReader{Module: forgeBridgeModule(), Call: keptCall, Logf: logger.Printf, Hub: hub} src := sourcebranches.New(sourcebranches.Call(keptCall)) @@ -191,7 +211,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // What the gate says it owes, on its seat's stream owed through this module's durable there (made from its consumes). // A stream not there yet is said, and the bridge goes on: its other work does not wait on the gate. if cc, err := cred.HearSeatEvents(ctx, js, seat.StreamOwed, func(m jetstream.Msg) { - hctx, cancel := context.WithTimeout(ctx, handleDeadline) + hctx, cancel := context.WithTimeout(ctx, awayBound+handleDeadline) // an act waits out a part away until owed-away-bound defer cancel() owe.hear(hctx, seat.Seat, m) }, logger.Printf); err != nil { @@ -205,7 +225,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro backlog := owedBacklog(js, cred, time.Now) go func() { err := watch(ctx, kv, func(key string, rev uint64, value []byte) { - hctx, cancel := context.WithTimeout(ctx, handleDeadline) + hctx, cancel := context.WithTimeout(ctx, awayBound+handleDeadline) // each act performed again is a kept call defer cancel() owe.asked(hctx, kv, key, rev, value, backlog) }) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go index 80475ec05..088996b86 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go @@ -30,6 +30,7 @@ func (f *fakeMsg) Subject() string { return f.subject } func (f *fakeMsg) Data() []byte { return f.data } func (f *fakeMsg) Headers() nats.Header { return nats.Header{} } func (f *fakeMsg) Ack() error { f.acked++; return nil } +func (f *fakeMsg) InProgress() error { return nil } func (f *fakeMsg) NakWithDelay(d time.Duration) error { f.nak = append(f.nak, d); return nil } func (f *fakeMsg) Term() error { f.termd++; return nil } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go index 8999e70a2..8a999ed9f 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go @@ -6,6 +6,8 @@ import ( "errors" "fmt" "log" + "sort" + "strings" "time" "github.com/nats-io/nats.go/jetstream" @@ -14,6 +16,7 @@ import ( seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-forge-bridge/internal/bridge" @@ -30,13 +33,16 @@ func performedKey(of, id string) string { return groups.PerformedKey(of, id) } // Performed is one value of the state performed. type Performed = groups.Performed -// performedStore is the state performed as the owing reads and writes it. +// performedStore is the state performed as the owing reads and writes it; Keys lists it for bridge_owed_again by record. type performedStore interface { Get(ctx context.Context, key string) (value []byte, found bool, err error) Put(ctx context.Context, key string, value []byte) error + Keys(ctx context.Context) ([]string, error) } -type kvPerformed struct{ kv keptStore } +type kvPerformed struct{ kv *boundBucket } + +func (s kvPerformed) Keys(ctx context.Context) ([]string, error) { return s.kv.Keys(ctx) } func (s kvPerformed) Get(ctx context.Context, key string) ([]byte, bool, error) { e, err := s.kv.Get(ctx, key) @@ -64,9 +70,12 @@ type owing struct { // acts reads an act again from the stream owed of the seat that said it (H-A-181), for bridge_owed_again. acts actReader // waits, when set, says why the state performed cannot be used yet (its binding, design 47 G §577); empty when it can. - waits func() string - now func() time.Time - logger *log.Logger + waits func() string + // awayBound is owed-away-bound: how long an act is called again through transport faults and the seat git's `away` + // before it ends could not be performed, naming its tries (design 47 H2 §807, §775); zero is busretry.Bound. + awayBound time.Duration + now func() time.Time + logger *log.Logger } // readerOf is the reading of the seat of's owed event into the seat git's verb. @@ -80,15 +89,17 @@ func readerOf(of string) (func([]byte) (bridge.OwedCall, error), error) { // heard performs one owed event the seat of said. Its error is only an outcome that could not be said on the stream // forge: the event is then heard again, and the act, safe to repeat, performed again. -func (o *owing) heard(ctx context.Context, of string, data []byte) (forgemsg.OwedEnded, error) { +// tried, when set, is told before each try of the act's kept call (the message's progress, so the bus does not hand it +// again while the act waits out a part away). +func (o *owing) heard(ctx context.Context, of string, data []byte, tried func()) (forgemsg.OwedEnded, error) { read, err := readerOf(of) if err != nil { - return o.end(ctx, forgemsg.OwedEnded{Of: of, Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, "", 0) + return o.end(ctx, forgemsg.OwedEnded{Of: of, Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, bridge.OwedCall{}) } c, err := read(data) if err != nil { return o.end(ctx, forgemsg.OwedEnded{Of: of, ID: c.ID, - Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, c.Repository, c.Number) + Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, c) } if w := o.waiting(); w != "" { return forgemsg.OwedEnded{Of: c.Of, ID: c.ID}, fmt.Errorf("the bridge %s", w) @@ -97,13 +108,13 @@ func (o *owing) heard(ctx context.Context, of string, data []byte) (forgemsg.Owe // said twice, or said again while its outcome was on its way: performed once, the outcome said once return forgemsg.OwedEnded{Of: c.Of, ID: c.ID}, nil } - return o.perform(ctx, c) + return o.perform(ctx, c, tried) } // hear is one message of a seat's stream owed: performed and acknowledged, or, when its outcome could not be said, // left to come again (design 47: the bridge stopped between an act and its owed.ended performs it again). func (o *owing) hear(ctx context.Context, of string, m delivered) { - if _, err := o.heard(ctx, of, m.Data()); err != nil { + if _, err := o.heard(ctx, of, m.Data(), func() { _ = m.InProgress() }); err != nil { o.logger.Printf("%s, so the act is heard again: %v", m.Subject(), err) _ = m.NakWithDelay(retryDelay) return @@ -119,21 +130,77 @@ func (o *owing) waiting() string { return o.waits() } -// perform calls the seat git's verb of one act and says what it came to. -func (o *owing) perform(ctx context.Context, c bridge.OwedCall) (forgemsg.OwedEnded, error) { - answer, callErr := o.call(ctx, "seat:"+git.Seat+"."+c.Verb, c.Input) - outcome, words := bridge.Outcome(answer, callErr) +// perform calls the seat git's verb of one act as a kept call (design 47 H2 §807, §775) and says what it came to: through +// transport faults and the seat's `away` it is called again at the kept-call pace until owed-away-bound, then ends could +// not be performed naming every try. Before each try its precondition is read again (H-A-432 (2), (3)): an act whose +// pull request closed unmerged ends not-needed, never performed. +func (o *owing) perform(ctx context.Context, c bridge.OwedCall, tried func()) (forgemsg.OwedEnded, error) { + type try struct { + answer json.RawMessage + closed string + } + t, callErr := busretry.DoPaced(busretry.Kept{Seat: git.Seat, Verb: c.Verb, Ask: c.ID, Bound: o.awayBound, Logf: o.logger.Printf}, func() (try, error) { + if tried != nil { + tried() + } + closed, err := o.closed(ctx, c) + if err != nil || closed != "" { + return try{closed: closed}, err + } + answer, err := o.call(ctx, "seat:"+git.Seat+"."+c.Verb, c.Input) + if err == nil { + if away := busretry.AwayIn(answer); away != nil { + return try{answer: answer}, away + } + } + return try{answer: answer}, err + }) + outcome, words := bridge.Outcome(t.answer, callErr) + if callErr == nil && t.closed != "" { + outcome, words = forgemsg.OwedNotNeeded, t.closed + } if outcome != forgemsg.OwedPerformed { o.logger.Printf("the owed %s %s of %s#%d (said by %s) is %s: %s", c.Act, c.ID, c.Repository, c.Number, c.Of, outcome, words) } return o.end(ctx, forgemsg.OwedEnded{Of: c.Of, ID: c.ID, - Outcome: outcome, Words: words}, c.Repository, c.Number) + Outcome: outcome, Words: words}, c) +} + +// closed is why the act is not needed: its pull request closed unmerged, as the seat git's pull answers it now (open +// and merged both need it, since the gate owes notes and views on merged pull requests; H-A-432 (2)); empty when it is +// needed. An act that names no pull request (a status said on `.0`) has none to read. A pull the forge does not answer +// is the try's transport fault, called again; one it refuses is the act's error, never performed unread. +func (o *owing) closed(ctx context.Context, c bridge.OwedCall) (string, error) { + if c.Number <= 0 { + return "", nil + } + raw, err := o.call(ctx, "seat:"+git.Seat+"."+git.VerbPull, git.PullInput{Repository: c.Repository, Number: int64(c.Number)}) + if err == nil { + if away := busretry.AwayIn(raw); away != nil { + err = away + } + } + if err != nil { + return "", fmt.Errorf("the pull request %s#%d could not be read before the act: %w", c.Repository, c.Number, err) + } + var a git.PullAnswer + switch { + case json.Unmarshal(raw, &a) != nil: + return "", fmt.Errorf("the seat git's pull of %s#%d answered what its contract cannot read: %s", c.Repository, c.Number, raw) + case a.Error != "": + return "", fmt.Errorf("the pull request %s#%d could not be read before the act: %s", c.Repository, c.Number, a.Error) + case a.Pull == nil: + return "", fmt.Errorf("the seat git's pull of %s#%d answered no pull request", c.Repository, c.Number) + case a.Pull.State == git.PullStateClosed: + return fmt.Sprintf("the pull request %s#%d closed unmerged, so the %s is not needed", c.Repository, c.Number, c.Act), nil + } + return "", nil } // end says an act's outcome as owed.ended on the stream forge, then keeps it in performed with the pull request the act -// names (repository and number). An outcome not kept is logged: the act may be performed again, which changes nothing at -// the forge. -func (o *owing) end(ctx context.Context, e forgemsg.OwedEnded, repository string, number int) (forgemsg.OwedEnded, error) { +// names (repository and number) and its delivery record. An outcome not kept is logged: the act may be performed again, +// which changes nothing at the forge. +func (o *owing) end(ctx context.Context, e forgemsg.OwedEnded, c bridge.OwedCall) (forgemsg.OwedEnded, error) { e.At = o.now().UTC() data, err := forgemsg.EncodeOwedEnded(e) if err != nil { @@ -145,7 +212,7 @@ func (o *owing) end(ctx context.Context, e forgemsg.OwedEnded, repository string if e.ID == "" { return e, nil // an undecodable act with no id is said, and has no key to keep } - v, err := json.Marshal(Performed{Ended: e, Repository: repository, Number: number}) + v, err := json.Marshal(Performed{Ended: e, Repository: c.Repository, Number: c.Number, Record: c.Record}) if err == nil { err = o.performed.Put(ctx, performedKey(e.Of, e.ID), v) } @@ -179,14 +246,19 @@ func (o *owing) kept(ctx context.Context, of, id string) (Performed, error) { return p, nil } -// answer answers one owed ask: an act performed again now, whatever performed holds of it; an act dropped, said as -// owed.ended with the outcome dropped and the ask's why (the tool wrote the hand-act log first); or the consumers' backlog. +// answer answers one owed ask: an act performed again now, whatever performed holds of it; every act of one delivery +// record that ended could not be performed, performed again (design 47 H2 §807, H-A-432 (1)); an act dropped, said as +// owed.ended with the outcome dropped and the ask's why (the tool wrote the hand-act log first); or the consumers' +// backlog. Each act performed again reads its precondition again before it acts (perform). func (o *owing) answer(ctx context.Context, a groups.OwedAsk, backlog func(context.Context) []groups.Backlog) groups.OwedAsk { a.Answered = o.now().UTC() - switch a.Kind { - case groups.OwedBacklog: + switch { + case a.Kind == groups.OwedBacklog: a.Backlog, a.Answer = backlog(ctx), "the backlog of each seat's stream owed" return a + case a.Kind == groups.OwedAgain && a.Record != "": + a.Answer = o.againRecord(ctx, a.Record) + return a } p, err := o.kept(ctx, a.Of, a.ID) if err != nil { @@ -195,15 +267,9 @@ func (o *owing) answer(ctx context.Context, a groups.OwedAsk, backlog func(conte } var e forgemsg.OwedEnded if a.Kind == groups.OwedDrop { - e, err = o.end(ctx, forgemsg.OwedEnded{Of: a.Of, ID: a.ID, Outcome: forgemsg.OwedDropped, Words: a.Why + " (dropped by " + a.By + ")"}, p.Repository, p.Number) - } else if read, rerr := readerOf(a.Of); rerr != nil { - err = rerr - } else if data, rerr := o.actOf(ctx, a.Of, p, a.ID); rerr != nil { - err = rerr - } else if c, rerr := read(data); rerr != nil { - e, err = o.end(ctx, forgemsg.OwedEnded{Of: a.Of, ID: a.ID, Outcome: forgemsg.OwedUndecodable, Words: rerr.Error()}, p.Repository, p.Number) + e, err = o.end(ctx, forgemsg.OwedEnded{Of: a.Of, ID: a.ID, Outcome: forgemsg.OwedDropped, Words: a.Why + " (dropped by " + a.By + ")"}, callOf(p)) } else { - e, err = o.perform(ctx, c) + e, err = o.again(ctx, a.Of, a.ID, p) } if err != nil { a.Answer = err.Error() @@ -216,6 +282,68 @@ func (o *owing) answer(ctx context.Context, a groups.OwedAsk, backlog func(conte return a } +// callOf is what performed holds of an act as the call an outcome is kept with: its pull request and its record. +func callOf(p Performed) bridge.OwedCall { + return bridge.OwedCall{Repository: p.Repository, Number: p.Number, Record: p.Record} +} + +// again performs one act of of again, read from its seat's stream owed on the pull request p names. +func (o *owing) again(ctx context.Context, of, id string, p Performed) (forgemsg.OwedEnded, error) { + read, err := readerOf(of) + if err != nil { + return forgemsg.OwedEnded{}, err + } + data, err := o.actOf(ctx, of, p, id) + if err != nil { + return forgemsg.OwedEnded{}, err + } + c, err := read(data) + if err != nil { + return o.end(ctx, forgemsg.OwedEnded{Of: of, ID: id, Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, callOf(p)) + } + if c.Record == "" { + c.Record = p.Record // the record kept with the act stays with it + } + return o.perform(ctx, c, nil) +} + +// againRecord performs again every act of the delivery record (its source revision) that performed holds as ended could +// not be performed, oldest first, and answers what each came to; a record none of whose acts so ended is said. +func (o *owing) againRecord(ctx context.Context, record string) string { + keys, err := o.performed.Keys(ctx) + if err != nil { + return "performed cannot be listed, so no act of the record " + record + " is performed again: " + err.Error() + } + var acts []Performed + for _, k := range keys { + v, found, err := o.performed.Get(ctx, k) + var p Performed + if err != nil || !found || json.Unmarshal(v, &p) != nil { + continue + } + if p.Record == record && p.Ended.Outcome == forgemsg.OwedCouldNotBePerformed { + acts = append(acts, p) + } + } + if len(acts) == 0 { + return "the record " + record + " holds no act that ended " + forgemsg.OwedCouldNotBePerformed + } + sort.Slice(acts, func(i, j int) bool { return acts[i].Ended.At.Before(acts[j].Ended.At) }) + lines := make([]string, 0, len(acts)) + for _, p := range acts { + e, err := o.again(ctx, p.Ended.Of, p.Ended.ID, p) + switch { + case err != nil: + lines = append(lines, p.Ended.ID+": "+err.Error()) + case e.Words != "": + lines = append(lines, p.Ended.ID+": "+e.Outcome+": "+e.Words) + default: + lines = append(lines, p.Ended.ID+": "+e.Outcome) + } + } + return fmt.Sprintf("%d acts of the record %s performed again: %s", len(acts), record, strings.Join(lines, "; ")) +} + // asked answers one value of the state owed-asks not yet answered, written back by compare-and-set at its revision; an // answered one (this bridge's own write) is left. func (o *owing) asked(ctx context.Context, kv jetstream.KeyValue, key string, rev uint64, value []byte, backlog func(context.Context) []groups.Backlog) { diff --git a/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go b/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go index 1a4c65481..1d576804a 100644 --- a/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go +++ b/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go @@ -96,6 +96,9 @@ func TestForge(t *testing.T) { {"no id", func(e *OwedEnded) { e.ID = "" }, "no id"}, {"a refusal with no words", func(e *OwedEnded) { e.Outcome = OwedRefused }, "no words"}, {"dropped with no reason", func(e *OwedEnded) { e.Outcome = OwedDropped }, "no words"}, + // Design 47 H2 §807 (H-A-432 (2)): an act whose pull request closed unmerged is said not-needed, naming it. + {"not needed, naming why", func(e *OwedEnded) { e.Outcome, e.Words = OwedNotNeeded, "novox/x#9 closed unmerged" }, ""}, + {"not needed with no words", func(e *OwedEnded) { e.Outcome = OwedNotNeeded }, "no words"}, {"no time", func(e *OwedEnded) { e.At = time.Time{} }, "no time"}, {"undecodable with no id is said", func(e *OwedEnded) { e.ID, e.Outcome, e.Words = "", OwedUndecodable, "not json" }, ""}, } { diff --git a/modules/mesh-delivery-forge-bridge/forgemsg/owed.go b/modules/mesh-delivery-forge-bridge/forgemsg/owed.go index e67e3bbec..4fd508efb 100644 --- a/modules/mesh-delivery-forge-bridge/forgemsg/owed.go +++ b/modules/mesh-delivery-forge-bridge/forgemsg/owed.go @@ -13,18 +13,20 @@ import ( const OwedEndedEvent = "owed.ended" // The outcomes of an owed act (design 47 G §164, §168): performed, refused (the forge's own words), could not be -// performed (the forge did not answer within the call's bound, with the error), undecodable (an act the bridge cannot -// read, with why) and, from the tool `bridge_owed_drop`, dropped. +// performed (the forge did not answer within the bridge's owed-away-bound, naming every try, or answered an error), +// undecodable (an act the bridge cannot read, with why), from the tool `bridge_owed_drop`, dropped, and not-needed (the +// act's pull request closed unmerged, read before a try, so it is not performed; design 47 H2 §807, H-A-432 (2)). const ( OwedPerformed = "performed" OwedRefused = "refused" OwedCouldNotBePerformed = "could not be performed" OwedUndecodable = "undecodable" OwedDropped = "dropped" + OwedNotNeeded = "not-needed" ) // OwedOutcomes are the outcomes an owed.ended says, one of them. -var OwedOutcomes = []string{OwedPerformed, OwedRefused, OwedCouldNotBePerformed, OwedUndecodable, OwedDropped} +var OwedOutcomes = []string{OwedPerformed, OwedRefused, OwedCouldNotBePerformed, OwedUndecodable, OwedDropped, OwedNotNeeded} // OwedEnded is an owed act's outcome (design 47 G §164): the act's id and one outcome, with the words of any outcome // but performed. Of is the seat that said the act (mesh-delivery for the gate's), so it hears its own ids (§164). An act the bridge cannot decode may name no id. diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/owed.go b/modules/mesh-delivery-forge-bridge/internal/bridge/owed.go index 4fc6d2ed4..739aee121 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/owed.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/owed.go @@ -19,8 +19,11 @@ type OwedCall struct { Act string Repository string Number int - Verb string - Input any + // Record is the delivery record the act is for, its source revision (design 47 H2 §807, H-A-432 (1)); empty when the + // owed named none. + Record string + Verb string + Input any } // ErrUndecodable is an owed event the bridge cannot read: it is said as owed.ended `undecodable`, never only logged. @@ -36,7 +39,7 @@ func GateOwed(data []byte) (OwedCall, error) { if err := json.Unmarshal(data, &o); err != nil { return OwedCall{}, undecodable("the gate's owed is not json: %v", err) } - c := OwedCall{Of: seat.Seat, ID: o.ID, Act: o.Act, Repository: o.Repository, Number: int(o.Number)} + c := OwedCall{Of: seat.Seat, ID: o.ID, Act: o.Act, Repository: o.Repository, Number: int(o.Number), Record: o.Record} if err := c.common(o.Head); err != nil { return c, err } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/owed_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/owed_test.go index e59b80e5f..9e3e8b076 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/owed_test.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/owed_test.go @@ -62,6 +62,13 @@ func TestAnOwedEventIsTheSeatGitsVerbAndItsAnswerOneOutcome(t *testing.T) { t.Errorf("%s: %s %+v %v, want %s %+v", c.name, got.Verb, got.Input, err, c.verb, c.input) } } + // Design 47 H2 §807 (H-A-432 (1)): the delivery record an owed act names is carried with it; an act naming none has none. + if got, err := GateOwed(edit(gate("owed is said, a status"), "record", "main.212")); err != nil || got.Record != "main.212" { + t.Errorf("an act of a record: %+v %v, want its record main.212", got, err) + } + if got, err := GateOwed(gate("owed is said, a status")); err != nil || got.Record != "" { + t.Errorf("an act of no record: %+v %v, want none", got, err) + } answers := func(verb string) []json.RawMessage { return gitfixture.All(t, verb+" answers", "verb:"+verb+".answer") } failed := func(verb string) json.RawMessage { return gitfixture.Given(t, verb+" answers an error, never none", "verb:"+verb+".answer") diff --git a/modules/mesh-delivery-forge-bridge/internal/groups/owedask.go b/modules/mesh-delivery-forge-bridge/internal/groups/owedask.go index 02e4d96a4..7f436ba56 100644 --- a/modules/mesh-delivery-forge-bridge/internal/groups/owedask.go +++ b/modules/mesh-delivery-forge-bridge/internal/groups/owedask.go @@ -46,9 +46,12 @@ type Backlog struct { // and once the bridge answered, when, what it came to (the outcome said as owed.ended) and its words; a backlog ask is // answered with the consumers' backlog. type OwedAsk struct { - Kind string `json:"kind"` - Of string `json:"of,omitempty"` - ID string `json:"id,omitempty"` + Kind string `json:"kind"` + Of string `json:"of,omitempty"` + ID string `json:"id,omitempty"` + // Record, for again instead of an act, is a delivery record's source revision: every act of it that ended could not + // be performed is performed again (design 47 H2 §807, H-A-432 (1)). + Record string `json:"record,omitempty"` Why string `json:"why,omitempty"` By string `json:"by,omitempty"` At time.Time `json:"at"` @@ -84,6 +87,20 @@ func NewOwedAsk(kind, of, id, why, by string, at time.Time) (string, []byte, err return key, v, err } +// NewOwedRecordAsk is the key and value of an ask to perform again every act of one delivery record (its source revision, +// .) that ended could not be performed: under again.record and the record (unpadded base64url). It says why. +func NewOwedRecordAsk(record, why, by string, at time.Time) (string, []byte, error) { + a := OwedAsk{Kind: OwedAgain, Record: strings.TrimSpace(record), Why: strings.TrimSpace(why), By: by, At: at.UTC()} + if a.Record == "" { + return "", nil, fmt.Errorf("a record is named by its source revision, .") + } + if a.Why == "" { + return "", nil, fmt.Errorf("performing a record's acts again says why") + } + v, err := json.Marshal(a) + return OwedAgain + ".record." + base64.RawURLEncoding.EncodeToString([]byte(a.Record)), v, err +} + // DecodeOwedAsk reads a value of owed-asks. func DecodeOwedAsk(data []byte) (OwedAsk, error) { var a OwedAsk diff --git a/modules/mesh-delivery-forge-bridge/internal/groups/performed.go b/modules/mesh-delivery-forge-bridge/internal/groups/performed.go index eaaa09fe5..81ea61f66 100644 --- a/modules/mesh-delivery-forge-bridge/internal/groups/performed.go +++ b/modules/mesh-delivery-forge-bridge/internal/groups/performed.go @@ -16,11 +16,14 @@ func PerformedKey(of, id string) string { return of + "." + base64.RawURLEncoding.EncodeToString([]byte(id)) } -// Performed is one value of the state performed (§163, H-A-182): the outcome said as owed.ended, with when, and the +// Performed is one value of the state performed (§163, H-A-182): the outcome said as owed.ended, with when, the // repository and number of the pull request the act names, so bridge_owed reads one pull request's acts and -// bridge_owed_again finds the act on its pull request's subject of the stream owed (H-A-181). +// bridge_owed_again finds the act on its pull request's subject of the stream owed (H-A-181), and the delivery record +// the act is for (its source revision, .), so bridge_owed_again by record finds the record's acts in +// this state (design 47 H2 §807, H-A-432 (1)); absent for an act whose owed named none. type Performed struct { Ended forgemsg.OwedEnded `json:"ended"` Repository string `json:"repository,omitempty"` Number int `json:"number,omitempty"` + Record string `json:"record,omitempty"` } diff --git a/modules/mesh-delivery-forge-bridge/manifest_test.go b/modules/mesh-delivery-forge-bridge/manifest_test.go index 90bd18e1e..19041986d 100644 --- a/modules/mesh-delivery-forge-bridge/manifest_test.go +++ b/modules/mesh-delivery-forge-bridge/manifest_test.go @@ -106,6 +106,7 @@ func sorted(s []string) []string { } func TestManifestDeclarations(t *testing.T) { // 1235/AC1 + t.Run("the process", manifestProcess) // design 47 H2 §819: one top-level test fewer m, top := load(t) if m.Module != "mesh-delivery-forge-bridge" { t.Errorf("module = %q", m.Module) @@ -127,8 +128,9 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 if len(m.Reads) != 0 { t.Errorf("reads = %v, want none", m.Reads) } - // No pull, tree, file, open pull requests, merge or group is asked (design 47 H §739, §764). - if want := []string{"seat:git.branch", "seat:git.comment", "seat:git.note", "seat:git.status", "seat:git.tags", "seat:git.view", + // No tree, file, open pull requests, merge or group is asked (design 47 H §739, §764); one pull request's state is read + // before each try of an act (design 47 H2 §807, H-A-432 (2)). + if want := []string{"seat:git.branch", "seat:git.comment", "seat:git.note", "seat:git.pull", "seat:git.status", "seat:git.tags", "seat:git.view", "seat:mesh-broker.show", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source", "seat:mesh-hand-acts.hand-act"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } @@ -155,8 +157,10 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 t.Errorf("tools = %v, want %v", m.Tools, want) } // performed's prune starts past prune-mark and ends at prune-target, settings read at run time (§812). - for key, def := range map[string]float64{"prune-mark": 70, "prune-target": 50} { - if c := m.Config[key]; c.Default != any(def) || !c.Runtime { + // Each owed act is a kept call at the module's pace until owed-away-bound (§807, §775). + for key, def := range map[string]any{"prune-mark": float64(70), "prune-target": float64(50), "owed-away-bound": "60m", + "kept-call-pace-first": "1s", "kept-call-pace-max": "30s"} { + if c := m.Config[key]; c.Default != def || !c.Runtime { t.Errorf("config %s = %+v, want a runtime setting defaulting to %v", key, c, def) } } @@ -183,7 +187,7 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 } } -func TestManifestProcess(t *testing.T) { // 1235/AC1 +func manifestProcess(t *testing.T) { // 1235/AC1 m, _ := load(t) var procs []map[string]any for _, r := range m.Resources { diff --git a/modules/mesh-delivery-forge-bridge/module.json b/modules/mesh-delivery-forge-bridge/module.json index b5a412112..0576160e0 100644 --- a/modules/mesh-delivery-forge-bridge/module.json +++ b/modules/mesh-delivery-forge-bridge/module.json @@ -30,6 +30,7 @@ "seat:git.status", "seat:git.comment", "seat:git.view", + "seat:git.pull", "seat:git.note", "seat:mesh-hand-acts.hand-act", "seat:git.tags", @@ -47,6 +48,27 @@ } ], "config": { + "owed-away-bound": { + "type": "string", + "description": "How long the bridge calls an owed act again through transport faults and the seat git's away before it says owed.ended could not be performed, naming its tries, a Go duration (design 47 H2 §807).", + "trusted": false, + "runtime": true, + "default": "60m" + }, + "kept-call-pace-first": { + "type": "string", + "description": "the first wait before a kept call met by a transport fault is made again, doubled per call up to kept-call-pace-max, a Go duration (design 47 H2 §775).", + "trusted": false, + "runtime": true, + "default": "1s" + }, + "kept-call-pace-max": { + "type": "string", + "description": "the most a kept call's wait between calls grows to, a Go duration (design 47 H2 §775).", + "trusted": false, + "runtime": true, + "default": "30s" + }, "prune-mark": { "type": "number", "description": "The fill of the bridge's state performed, in percent of its cap, past which the bridge prunes it at its own write, oldest acts first (design 47 H2 §812): below the bus module's raise mark of 75 %.", From 7d58b2e56fbda6855ab8e855595c5ca98963c09d Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 02:06:47 +0100 Subject: [PATCH 12/12] H-A-441: the validate step takes the trunk's seat list from seat-list and names its digest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A revision that names no definitions of its own is judged by the trunk's seat list as the seat mesh-catalog's chunked verb seat-list hands it, asked as a kept call under the run's ask: its bytes fill Request.Seats and its digest is the request's seat-list, the bytes refused when their size or sha256 is not the answer's, so every validate run of one list judges the same bytes and the validator reads no bus. A revision naming its own definitions keeps them, sorted by seat, and names the sha256 of those bytes. The manifest invokes seat:mesh-catalog.seat-list. Builds: design 47 H2 §814; H-A-441, H-A-443 --- .../mesh-delivery-assigner/compositionrun.go | 4 + .../compositionstep_integration_test.go | 6 ++ .../integration_test.go | 36 +++++++- .../cmd/mesh-delivery-assigner/main.go | 2 +- .../mesh-delivery-assigner/validatestep.go | 85 ++++++++++++++++++- .../validatestep_test.go | 55 +++++++++++- .../mesh-delivery-assigner/manifest_test.go | 1 + modules/mesh-delivery-assigner/module.json | 1 + 8 files changed, 184 insertions(+), 6 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go index daddb921d..2188cd04a 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionrun.go @@ -22,6 +22,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-assigner/internal/assign" ) @@ -55,6 +56,9 @@ type compositionRun struct { keeper *inputKeeper // call asks a seat verb held once for the mesh (process.Conn.CallKept). call func(key string, body any) (json.RawMessage, error) + // callFile asks a seat verb held once for the mesh whose answer comes in chunks, as a kept call (process.Conn.CallFileKept), + // answering the verb's answer with the bytes received as its member `file`, base64: the validate step's seat-list (H-A-443). + callFile func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) // trunk is the trunk's catalog half as the assigner holds it. trunk func() (mcc.Half, bool) // newest is the newest catalog half the assigner's views hold of each watched branch (halves.views), and watched a module's diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go index 374f91e2e..51528de73 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/compositionstep_integration_test.go @@ -362,6 +362,12 @@ func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T if err != nil || len(req.Cases) != 2 || req.Cases[0].Name != changeCase || req.Cases[1].Name != beforeCase { t.Fatalf("the validator's request: %+v %v", req, err) } + // The revision names no definitions: the request holds the trunk's seat list as seat-list handed it, named by its digest (H-A-443). + if list, digest := seatListBytes(); req.SeatList != digest { + t.Errorf("the validator's request names the seat list %q, want %q", req.SeatList, digest) + } else if got, _ := json.Marshal(req.Seats); string(got) != string(list) { + t.Errorf("the validator's request holds seats %d bytes long, not the seat list's %d", len(got), len(list)) + } v := composer.Verdict{V: composer.Version} for _, c := range req.Cases { reads := map[string]composer.ManifestRead{} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 690041718..8358c76d8 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -50,6 +50,7 @@ import ( mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/announce" + "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/process" @@ -1941,7 +1942,9 @@ func awaitKept(t *testing.T, b *bed, stream string, n uint64) { // serveSeatList answers the seat mesh-catalog's verb seat-definitions as the indexer does (ADR 0636 §4): with no seat, every // trunk seat with its digest; with one, its definition resolved, from the mesh-sdk test copy of every definition. The channel -// it answers is closed once every listed seat's definition was asked, the end of a reader's first read. +// it answers is closed once every listed seat's definition was asked, the end of a reader's first read. It also answers the +// chunked read verb seat-list as the indexer does (design 47 H2 H-A-443): the definitions as one JSON array sorted by seat, +// its sha256 and size, then the bytes on the call's chunk subjects (seatListBytes). func serveSeatList(t *testing.T, nc *nats.Conn) <-chan struct{} { t.Helper() var mu sync.Mutex @@ -1984,9 +1987,40 @@ func serveSeatList(t *testing.T, nc *nats.Conn) <-chan struct{} { } t.Cleanup(func() { _ = sub.Unsubscribe() }) } + list, digest := seatListBytes() + hand := func(m *nats.Msg) { + var in meshcatalog.SeatListInput + _ = json.Unmarshal(m.Data, &in) + send := buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbSeatList, Call: in.Call} + chunks := int64((len(list) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes) + raw, _ := json.Marshal(map[string]any{"result": meshcatalog.SeatListAnswer{Digest: digest, Size: int64(len(list)), Chunks: chunks}, "node": "n0"}) + _ = m.Respond(raw) + go func() { _ = send.Send(nc, list, buscalls.SeatAckWithin, t.Logf) }() + } + for _, subject := range []string{"mesh.seat." + meshcatalog.Seat + ".tool." + meshcatalog.VerbSeatList, "mesh.seat." + meshcatalog.Seat + ".call." + meshcatalog.VerbSeatList + ".*"} { + sub, err := nc.Subscribe(subject, hand) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + } return read } +// seatListBytes are the bed's seat list as the verb seat-list hands it: every definition of the mesh-sdk test copy as one JSON +// array sorted by seat, and its sha256 (H-A-441 (a)). +func seatListBytes() ([]byte, string) { + defs := slices.Clone(seatlisttest.Definitions()) + slices.SortFunc(defs, func(a, b seatdef.Definition) int { return strings.Compare(a.Seat, b.Seat) }) + list := make([]json.RawMessage, 0, len(defs)) + for _, d := range defs { + raw, _ := json.Marshal(d) + list = append(list, raw) + } + raw, _ := json.Marshal(list) + return raw, seatListDigest(raw) +} + // seatQueue makes the seat mesh-wants' work queue and its one shared worker from the seat's definition, as the broker derives // them (design 47 H2 §777). func seatQueue(ctx context.Context, t *testing.T, js jetstream.JetStream) { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 71c2b5d21..d0fff5108 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -679,7 +679,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, if err != nil { return err } - run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFileKept, bound: askBound}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, + run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFileKept, bound: askBound}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, callFile: bus.CallFileKept, trunk: trunkOf(iv, w.trunkName), newest: iv.newestOf(), watched: iv.watchedOf, assigned: w.assignedOn, assignment: w.assignmentOf, findings: findings, running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf} // The seat's work queue (design 47 H2 §777): pulled through its one worker, each ask taken once in the state taken. takenBound, err := askTakenBound() diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep.go index d7bd437c1..c513565f1 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep.go @@ -2,6 +2,10 @@ package main import ( "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -16,6 +20,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-assigner/internal/buckets" ) @@ -283,7 +288,7 @@ func (r *compositionRun) startValidate(ctx context.Context, key string, run mesh return } program.Package = validatorPackage - seats, err := r.seats(ctx, half.Revision.Definitions) + seats, list, err := r.validateSeats(ctx, ask, half.Revision.Definitions) if err != nil { couldNot(err.Error()) return @@ -300,7 +305,7 @@ func (r *compositionRun) startValidate(ctx context.Context, key string, run mesh } // No predefined key list and no known package names until a design line names the verb that gives them (H-A-108 (2)). req := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: changeCase, Manifests: change}, {Name: beforeCase, Manifests: prior}}, - Seats: seats} + Seats: seats, SeatList: list} raw, err := json.Marshal(req) if err != nil { couldNot("its request cannot be encoded: " + err.Error()) @@ -410,3 +415,79 @@ func validateBeforeTools(st store) []stdio.Tool { return validateBefore(st, arg(args, "pipeline"), arg(args, "module")) }}} } + +// seatListKey is the seat mesh-catalog's chunked read verb seat-list (design 47 H2 §814, H-A-441, H-A-443). +const seatListKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbSeatList + +// validateSeats are the seat definitions a validate run judges by, as one JSON array in composer.Request.Seats' form sorted by +// seat, and their digest, the sha256 of those bytes (H-A-441, H-A-443): a revision that names its own definitions keeps them, +// each read through seat-definitions; one that names none takes the trunk's seat list, the bytes the seat mesh-catalog's verb +// seat-list hands, asked as a kept call under the run's ask. +func (r *compositionRun) validateSeats(ctx context.Context, ask string, defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, string, error) { + if len(defs) == 0 { + return r.seatList(ask) + } + sorted := slices.Clone(defs) + slices.SortStableFunc(sorted, func(a, b meshcatalog.CatalogRevisionDefinitionsItem) int { return strings.Compare(a.Seat, b.Seat) }) + seats, err := r.seats(ctx, sorted) + if err != nil { + return nil, "", err + } + raw, err := json.Marshal(seats) + if err != nil { + return nil, "", fmt.Errorf("the revision's seat definitions cannot be encoded: %w", err) + } + var list []json.RawMessage + if err := json.Unmarshal(raw, &list); err != nil { + return nil, "", fmt.Errorf("the revision's seat definitions do not read again: %w", err) + } + return list, seatListDigest(raw), nil +} + +// seatListDigest is the digest of a seat list's bytes, as the indexer names its object in seat-lists: the sha256, hex. +func seatListDigest(raw []byte) string { + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]) +} + +// seatList is the trunk's seat list as the verb seat-list hands it (H-A-443): its digest and size, then the object's bytes in +// chunks; bytes whose size or sha256 is not the answer's, or that are no JSON array, are refused, so every run of one digest +// judges the same bytes. +func (r *compositionRun) seatList(ask string) ([]json.RawMessage, string, error) { + what := "the trunk's seat list is not read through " + seatListKey + if r.callFile == nil { + return nil, "", errors.New(what + ": the assigner holds no chunked call") + } + var bits [16]byte + if _, err := rand.Read(bits[:]); err != nil { + return nil, "", fmt.Errorf("%s: %w", what, err) + } + raw, err := r.callFile(process.KeptAsk{ID: ask + "." + meshcatalog.VerbSeatList, Bound: readsBound}, seatListKey, + meshcatalog.SeatListInput{Call: hex.EncodeToString(bits[:])}) + if err != nil { + return nil, "", fmt.Errorf("%s: %w", what, err) + } + var a struct { + meshcatalog.SeatListAnswer + File string `json:"file"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, "", fmt.Errorf("%s: its answer does not read: %w", what, err) + } + if a.Digest == "" { + return nil, "", errors.New(what + ": its answer names no digest") + } + file, err := base64.StdEncoding.DecodeString(a.File) + if err != nil { + return nil, "", fmt.Errorf("%s: the bytes received are not base64: %w", what, err) + } + if int64(len(file)) != a.Size || seatListDigest(file) != a.Digest { + return nil, "", fmt.Errorf("%s: %d bytes of digest %s were received, the answer names %d bytes of digest %s", what, len(file), + seatListDigest(file), a.Size, a.Digest) + } + var list []json.RawMessage + if err := json.Unmarshal(file, &list); err != nil || len(list) == 0 { + return nil, "", fmt.Errorf("%s: the seat list %s is no JSON array of seat definitions", what, a.Digest) + } + return list, a.Digest, nil +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go index 4d7985b13..cbe180e69 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/validatestep_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/base64" "encoding/json" "errors" "slices" @@ -15,6 +16,7 @@ import ( meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" + "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-assigner/internal/buckets" ) @@ -63,19 +65,40 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) { if !ok || in.Branch != "main" { return nil, errors.New("asked another revision") } + // Revision 3 names no definitions of its own: it is judged by the trunk's seat list, the bytes seat-list hands (H-A-443 (2)). + var defs []meshcatalog.CatalogRevisionDefinitionsItem + if in.Revision != 3 { + defs = []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "sd"}} + } return json.Marshal(meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: in.Revision, Base: base(in.Revision), - Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "sd"}}}, Entries: entries}) + Definitions: defs}, Entries: entries}) case seatDefinitionsKey: return json.Marshal(meshcatalog.SeatDefinitionsAnswer{Found: true, Entry: &meshcatalog.SeatDefinitionsAnswerEntry{Seat: "s1", Digest: "sd", Definition: map[string]any{"seat": "s1"}}}) } return nil, errors.New("no such verb " + key) } + // The trunk's seat list as the verb seat-list hands it (H-A-443 (1)): digest and size, then the bytes, which CallFileKept adds as file. + trunkList := []byte(`[{"seat":"a-seat"},{"seat":"s1","v":2}]`) + listAnswer := func(file []byte, digest string, size int) json.RawMessage { + raw, _ := json.Marshal(map[string]any{"digest": digest, "size": size, "chunks": 1, "file": base64.StdEncoding.EncodeToString(file)}) + return raw + } + var listAsks []process.KeptAsk + listed := listAnswer(trunkList, seatListDigest(trunkList), len(trunkList)) + callFile := func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) { + in, _ := body.(meshcatalog.SeatListInput) + if key != seatListKey || len(in.Call) != 32 { + return nil, errors.New("asked " + key + " with the call " + in.Call) + } + listAsks = append(listAsks, ask) + return listed, nil + } var mu sync.Mutex var asks []buildstep.StepAsk var requests []composer.Request keeper := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now) runs := newCompositionRuns(newRunsMem(), defaultRunSettings, time.Now, t.Logf) - r := &compositionRun{runs: runs, keeper: keeper, reads: &compositionReads{}, call: call, now: time.Now, logf: t.Logf, + r := &compositionRun{runs: runs, keeper: keeper, reads: &compositionReads{}, call: call, callFile: callFile, now: time.Now, logf: t.Logf, put: func(_ context.Context, a buildstep.StepAsk) error { body, err := keeper.body(a.Run.Input.Digest) if err != nil { @@ -146,6 +169,11 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) { if c := requests[0].Cases; len(c) != 2 || c[0].Name != changeCase || c[1].Name != beforeCase || len(c[1].Manifests) != 3 || requests[0].Predefined != nil || requests[0].Packages != nil { t.Errorf("the validator's request is %+v", requests[0]) } + // A revision naming its own definitions keeps them, and the request names the sha256 of their bytes (H-A-443 (2)); no seat + // list is asked. + if own := []byte(`[{"seat":"s1"}]`); requests[0].SeatList != seatListDigest(own) || len(requests[0].Seats) != 1 || string(requests[0].Seats[0]) != `{"seat":"s1"}` || len(listAsks) != 0 { + t.Errorf("a revision naming its definitions was judged by %s %s (%d seat-list asks), want %s", requests[0].SeatList, requests[0].Seats, len(listAsks), seatListDigest(own)) + } if _, _, ok, _ := runs.read("main.7" + validateKeySuffix); !ok { t.Error("the validate run is not the key .validate of composition-runs") } @@ -158,6 +186,29 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) { o.Said != reasonManifestRefused+`: web: unknown field "colour" (`+emptyBeforeWords+")" || len(asks) != 2 || len(requests[1].Cases[1].Manifests) != 0 { t.Errorf("a revision naming no base ended %+v (%d validator asks), want it judged against an empty case before, said", o, len(asks)) } + // Revision 3 names no definitions: Seats are the trunk's seat list's bytes as seat-list handed them, named by its digest, + // asked once as a kept call under the run's ask (H-A-441 (b), H-A-443). + if got, _ := json.Marshal(requests[1].Seats); requests[1].SeatList != seatListDigest(trunkList) || string(got) != string(trunkList) || + len(listAsks) != 1 || listAsks[0].ID != asks[1].ID+"."+meshcatalog.VerbSeatList || listAsks[0].Bound <= 0 { + t.Errorf("a revision naming no definitions was judged by %s %s (asks %+v), want the trunk's seat list %s", requests[1].SeatList, got, listAsks, seatListDigest(trunkList)) + } + // Bytes the answer does not name, by size or by digest, or that are no seat list, are refused: the run judges no other bytes. + for _, row := range []struct { + name string + answer json.RawMessage + }{ + {"a digest of other bytes", listAnswer(trunkList, seatListDigest([]byte("[]")), len(trunkList))}, + {"another size", listAnswer(trunkList, seatListDigest(trunkList), len(trunkList)+1)}, + {"no digest", listAnswer(trunkList, "", len(trunkList))}, + {"no JSON array", listAnswer([]byte(`{"seat":"s1"}`), seatListDigest([]byte(`{"seat":"s1"}`)), len(`{"seat":"s1"}`))}, + {"an empty list", listAnswer([]byte(`[]`), seatListDigest([]byte(`[]`)), 2)}, + } { + listed = row.answer + if seats, digest, err := r.seatList("ask-row"); err == nil || !strings.Contains(err.Error(), seatListKey) { + t.Errorf("%s: the seat list read %s %s, %v; want it refused, naming the verb", row.name, digest, seats, err) + } + } + listed = listAnswer(trunkList, seatListDigest(trunkList), len(trunkList)) // Revision 12 holds revision 7's manifests and names base 7: judged against it, web's problem is the revision's, db's old one // is not, and the case before holds base 7's manifests. if o := r.validate(ctx, "ask-base", step("web", 12)); o.Outcome != meshwants.StepEndedOutcomeFailed || diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 03bde3065..ab95812aa 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -148,6 +148,7 @@ func TestManifest(t *testing.T) { if want := []string{"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", buckets.TellerModule + "." + buckets.AnswersReadTool, // A-175, A-178: the teller's answers' first view "seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list + "seat:mesh-catalog.seat-list", // design 47 H2 H-A-443: the validate step's trunk seat list, in chunks "seat:mesh-conditions.conditions", // design 47 G §706: the conditions read at each judging of an outside wait "seat:mesh-delivery.source", // §507: the branches the gate keeps a source revision of, before the first registrations "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 097af3f1f..b6a11c4b4 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -72,6 +72,7 @@ "seat:mesh-catalog.registrations", {"verb": "seat:mesh-catalog.revision", "fields": ["none-yet"]}, "seat:mesh-catalog.seat-definitions", + "seat:mesh-catalog.seat-list", "seat:mesh-conditions.conditions", "seat:mesh-delivery.source", "seat:mesh-hand-acts.hand-act",