A2: the assigner pulls the seat mesh-wants' work queue and takes each ask once

A step ask put while the assigner was down waited for nobody: the plain
subscription heard only what came while it ran. The assigner now pulls every
waiting ask from the seat's one worker, takes it once by its id in its state
taken (for ask-taken-bound), says ask-taken at the pull, sends in-progress
while the step runs and acknowledges once the step's end is said. It serves
the seat's act drop: a running ask is cancelled, one not yet pulled is kept in
taken as dropped.<ask>; either ends its step could not run naming the drop.
assigner_asks reads what waits on the queue (stdio.QueueWaiting) and what is
taken or dropped; assigner_ask_drop ends one ask, written in the hand-act log.
The manifest declares taken, drop, the two tools, ask-taken-bound and the
kept-call pace as runtime settings.

Builds: design 47 H2 §777, §775, §778, §820; H-A-381, H-A-401, H-A-404, H-A-410
This commit is contained in:
2026-11-22 01:05:07 +01:00
parent 9acf583daf
commit dc72860ce0
13 changed files with 955 additions and 89 deletions
@@ -0,0 +1,131 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
)
// The seat mesh-wants' work queue read and steered by hand (design 47 H2 §777, the tools table; H-A-410): assigner_asks
// answers what waits on the seat's queue (read by the runtime without taking it) and what the assigner took and keeps
// dropped in its state taken; assigner_ask_drop ends one ask through the seat's act drop, written in the hand-act log first.
const (
asksTool = "assigner_asks"
askDropTool = "assigner_ask_drop"
)
// asksAnswerMost is the most an asks answer carries of the taken keys (§811: at most 768 KB with capped and total).
const asksAnswerMost = 768 << 10
// takenReader is what assigner_asks reads of the state taken (stdio.KeptState).
type takenReader interface {
Keys() ([]string, error)
Get(key string) (*stdio.StateEntry, error)
}
// asksOf is assigner_asks' answer: the asks on the seat's queue, each waiting, waiting again or held, the ids taken (key mesh-wants.<ask>) and the drops of
// asks not pulled yet (dropped.<ask>), each list cut at the answer's bound with capped and total.
func asksOf(waiting func(seat string) (stdio.Waiting, error), taken takenReader) (map[string]any, error) {
w, err := waiting(meshwants.Seat)
if err != nil {
return nil, fmt.Errorf("the asks waiting on the seat %s's work queue are not read: %w", meshwants.Seat, err)
}
keys, err := taken.Keys()
if err != nil {
return nil, fmt.Errorf("the state %s is not listed: %w", process.TakenState, err)
}
slices.Sort(keys)
// An ask the queue handed out and not acknowledged (again) is held when this holder took it, else it waits again after a
// nak (H-A-410's addenda): the bus says what it knows, the state taken the rest.
size := 0
waits := make([]map[string]any, 0, len(w.Asks))
for _, a := range w.Asks {
size += len(a.ID) + len(a.Module) + 64
state := "waiting"
if a.Again {
state = "waiting again"
if slices.Contains(keys, process.TakenKey(meshwants.Seat, a.ID)) {
state = "held"
}
}
waits = append(waits, map[string]any{"id": a.ID, "module": a.Module, "age_seconds": a.AgeSeconds, "state": state})
}
takenIDs, dropped := []string{}, []map[string]any{}
total, capped := 0, w.Capped
for _, k := range keys {
if id, ok := strings.CutPrefix(k, process.TakenKey(meshwants.Seat, "")); ok {
total++
if size += len(id) + 4; size > asksAnswerMost {
capped = true
continue
}
takenIDs = append(takenIDs, id)
continue
}
id, ok := strings.CutPrefix(k, droppedPrefix)
if !ok {
continue
}
total++
e, err := taken.Get(k)
if err != nil || e == nil {
continue
}
var d askDrop
if json.Unmarshal(e.Value, &d) != nil {
continue
}
if size += len(id) + len(d.By) + len(d.Why) + 64; size > asksAnswerMost {
capped = true
continue
}
dropped = append(dropped, map[string]any{"id": id, "by": d.By, "why": d.Why, "at": d.At})
}
return map[string]any{"waiting": waits, "taken": takenIDs, "dropped": dropped, "capped": capped, "total": total + len(w.Asks)}, nil
}
// asksTools are the module's tools of the seat's work queue.
func asksTools(waiting func(string) (stdio.Waiting, error), taken takenReader, ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
return []stdio.Tool{
{Name: asksTool, Description: "The asks of the seat mesh-wants' work queue (design 47 H2 §777): each one on it (its id, the module " +
"that asked it, its age and whether it waits, waits again after a hand-back or is held here), the ids the assigner took (kept for ask-taken-bound) and the drops of asks not pulled yet, " +
"cut at 768 KB with capped and total. Read-only.",
Input: schema(map[string]string{}, nil),
Run: func(map[string]any) (any, error) {
return asksOf(waiting, taken)
}},
{Name: askDropTool, Description: "End one ask of the seat mesh-wants by hand (design 47 H2 §777, §778): written in the hand-act log " +
"first, then the seat's act drop ends it: one running is cancelled, one waiting is ended when it is pulled; either ends " +
"its step could not run naming the drop and who asked it.",
Input: schema(map[string]string{"ask": "the ask's id", "why": "why the ask is ended"}, []string{"ask", "why"}),
Run: func(args map[string]any) (any, error) {
id, why := strings.TrimSpace(arg(args, "ask")), strings.TrimSpace(arg(args, "why"))
if id == "" || why == "" {
return nil, errors.New("name the ask and why it is ended")
}
by := stdio.CallerOf(args)
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: askDropTool + " " + id, Why: why + " (" + by + ")", Cause: "ask-drop"})
if err != nil {
return nil, fmt.Errorf("the hand act was not recorded, so nothing was dropped: %w", err)
}
raw, err := ask("seat:"+meshwants.Seat+"."+meshwants.VerbDrop, meshwants.DropInput{ID: id, By: askDropTool + " (" + by + ")", Why: why})
if err != nil {
return nil, err
}
var out map[string]any
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("%s answered what is not JSON: %w", meshwants.VerbDrop, err)
}
out["hand-act"] = taken.Log
return out, nil
}},
}
}
@@ -638,7 +638,7 @@ func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) {
}
got := map[string]meshwants.StepEnded{}
for _, m := range []string{"a", "b", "d"} {
steps.take(context.Background(), "ask-"+m, step(m, false))
pullOne(steps, "ask-"+m, step(m, false))
e := <-ends
got[e.Module] = e
}
@@ -649,7 +649,7 @@ func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) {
mu.Lock()
fixed = true
mu.Unlock()
steps.take(context.Background(), "ask-a2", step("a", true))
pullOne(steps, "ask-a2", step("a", true))
moved := map[string]meshwants.StepEnded{}
for range 2 {
select {
@@ -126,7 +126,7 @@ func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) {
from := b.journal.mark()
_, stop := b.startStoppable(t)
b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second)
b.journal.await(t, from, "the composition step is pulled from", "the composition step served", 30*time.Second)
select {
case <-b.seatListRead:
case <-time.After(30 * time.Second):
@@ -2,14 +2,11 @@ package main
import (
"context"
"encoding/json"
"errors"
"sync"
"time"
"github.com/nats-io/nats.go"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/process"
)
// The composition step (design 47 G §22, §354, §63): the assigner takes the seat mesh-wants' accepted kind `step`, the
@@ -44,9 +41,18 @@ type compositionSteps struct {
logf func(string, ...any)
// ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own.
ticks func(d time.Duration) (<-chan time.Time, func())
mu sync.Mutex
// stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave).
stops map[string]context.CancelFunc
// taken is the state taken (stepqueue.go): each ask pulled is taken once by its id, for takenBound; the drops of asks not
// pulled yet are kept there too (H-A-404).
taken takenStore
takenBound time.Duration
// inProgress is how often a running step's pulled ask is said in progress: the seat definition's in-progress (§777).
inProgress func() time.Duration
mu sync.Mutex
// stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave), or the ask is
// dropped (the cause an *askDrop).
stops map[string]context.CancelCauseFunc
// held are the messages pulled from the seat's work queue for each running step's ask, acknowledged once its end is said.
held map[string][]queueMsg
}
// progressEvery is how often a running composition step says its progress (design 47 G §496, A-262).
@@ -92,20 +98,6 @@ func (p *stepPhase) get() string {
return p.phase
}
// take takes one ask of the accepted kind `step` by its id and body: a step that is no composition or validate step, or an
// ask with no id, is left with a log line; one taken is run in its own goroutine and its end said.
func (c *compositionSteps) take(ctx context.Context, id string, data []byte) bool {
var run meshwants.StepRun
err := json.Unmarshal(data, &run)
how := c.howOf(run.Step, false)
if err != nil || id == "" || how == nil {
c.logf("a step ask that is no composition or validate step with an id is left: %v", err)
return false
}
go c.say1(ctx, id, run, how, "")
return true
}
// howOf is what runs a step, or judges it once taken up at the start: by its step, composition or validate; nil for another.
func (c *compositionSteps) howOf(step string, resumed bool) func(context.Context, string, meshwants.StepRun) stepOutcome {
switch {
@@ -124,16 +116,16 @@ func (c *compositionSteps) howOf(step string, resumed bool) func(context.Context
// say1 runs one step with how and says its end, naming in `shared` the runs it read (§723); again is, for an end said again, the
// ask of the run whose end makes it say again (sayEnd). A step whose wait was ended by its pipeline's supersede or cancel (leave)
// says nothing: the runner ended it already.
func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome, again string) {
func (c *compositionSteps) say1(parent context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome, again string) {
started := c.now().UTC()
ctx, stopWait := context.WithCancel(ctx)
ctx, stopWait := context.WithCancelCause(parent)
c.mu.Lock()
if c.stops == nil {
c.stops = map[string]context.CancelFunc{}
c.stops = map[string]context.CancelCauseFunc{}
}
if c.stops[id] != nil { // the step asked under id is running already: it says its own end
c.mu.Unlock()
stopWait()
stopWait(nil)
return
}
c.stops[id] = stopWait
@@ -142,14 +134,25 @@ func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.St
c.mu.Lock()
delete(c.stops, id)
c.mu.Unlock()
stopWait()
stopWait(nil)
}()
phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch}
stop := c.sayProgress(id, run.Asker, phase)
stopInProgress := c.inProgressEvery(id)
o := how(context.WithValue(ctx, phaseKey{}, phase), id, run)
stopInProgress()
stop()
if ctx.Err() != nil {
if parent.Err() != nil { // the assigner stops: its pulled asks are handed again to its next start
return
}
var d *askDrop
if errors.As(context.Cause(ctx), &d) { // dropped through the seat's drop (§778, H-A-401)
c.sayEnd(id, run, &started, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: d.Error(), Shared: o.Shared}, again)
return
}
c.logf("the composition step %s of %s says no end: its wait ended (%v)", id, run.Pipeline, context.Cause(ctx))
c.settle(id, true) // the runner ended it already: its ask is over
return
}
c.sayEnd(id, run, &started, o, again)
@@ -175,9 +178,11 @@ func (c *compositionSteps) sayEnd(id string, run meshwants.StepRun, started *tim
msgID = id + "." + again
}
if err := c.say(meshwants.EventStepEnded+"."+c.node, end, msgID); err != nil {
c.logf("saying the end of the composition step %s: %v", id, err)
c.logf("saying the end of the composition step %s: %v; its ask is handed again", id, err)
c.settle(id, false)
return
}
c.settle(id, true) // the ask's end is said: its pulled messages leave the seat's work queue (§777)
if c.ended != nil {
c.ended(id, run, ended)
}
@@ -210,7 +215,7 @@ func (c *compositionSteps) leave(ids []string) {
defer c.mu.Unlock()
for _, id := range ids {
if stop := c.stops[id]; stop != nil {
stop()
stop(nil)
}
}
}
@@ -243,21 +248,3 @@ func (c *compositionSteps) sayProgress(id, asker string, phase *stepPhase) (stop
}()
return func() { close(done); <-ended }
}
// stepAskID is an ask's id as the asker names it: its event id, else its message id.
func stepAskID(m *nats.Msg) string {
if id := m.Header.Get("x-event-id"); id != "" {
return id
}
return m.Header.Get(nats.MsgIdHdr)
}
// serveCompositionSteps hears the composition step asks on the seat mesh-wants' accept subject.
func serveCompositionSteps(ctx context.Context, conn process.Subscriber, c *compositionSteps) error {
sub, err := conn.Subscribe(meshwants.AcceptSubjectStep, func(m *nats.Msg) { c.take(ctx, stepAskID(m), m.Data) })
if err != nil {
return err
}
go func() { <-ctx.Done(); _ = sub.Drain() }()
return nil
}
@@ -106,7 +106,7 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T)
from := b.journal.mark()
b.start(t)
b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second)
b.journal.await(t, from, "the composition step is pulled from", "the composition step served", 30*time.Second)
b.journal.await(t, from, "n3 is away (machine-away)", "the machine-away of n3 heard", 30*time.Second)
select {
@@ -114,14 +114,8 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T)
case <-time.After(30 * time.Second):
t.Fatal("the assigner did not read the seat list within 30 s")
}
step, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.3.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: trunkBranch,
putStep(b.ctx, t, b.js, "step-ask-1", meshwants.StepRun{Pipeline: "main.3.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: trunkBranch,
Revision: 3, Commit: engineCommit, Fingerprint: "src2:m1"})
msg := nats.NewMsg(meshwants.AcceptSubjectStep)
msg.Header.Set("x-event-id", "step-ask-1")
msg.Data = step
if err := nc.PublishMsg(msg); err != nil {
t.Fatal(err)
}
// The stand-in Go build seat takes the run asks put under the assigner's name, reads each request by the verb input and says
// its end with the composer's verdict: n1 composes, n2 does not.
@@ -201,6 +195,7 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T)
t.Fatalf("the inputs stay kept past their step-ended: %v", keys)
}
}
queueEmpty(b.ctx, t, b.js) // the step's ask acknowledged at its end (design 47 H2 §777)
}
// readInput reads a run's input as a build seat does: the seat mesh-wants' verb input answers its digest and chunk count, then its
@@ -288,9 +283,37 @@ func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T
}
t.Cleanup(func() { _ = ends.Unsubscribe() })
subscribed(t, nc)
// Design 47 H2 §777: a validate ask put while the assigner is stopped waits on the seat's work queue, and is taken (ask-taken)
// and ended after its start.
taken := make(chan *nats.Msg, 8)
takens, err := nc.ChanSubscribe(meshwants.SubjectAskTakenAll, taken)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = takens.Unsubscribe() })
subscribed(t, nc)
putStep(b.ctx, t, b.js, "validate-9", meshwants.StepRun{Pipeline: trunkBranch + ".9", Step: meshwants.StepRunStepValidate, Module: "a",
Branch: trunkBranch, Revision: 9, Commit: commit})
from := b.journal.mark()
b.start(t)
b.journal.await(t, from, "the composition step is heard on", "the steps served", 30*time.Second)
b.journal.await(t, from, "the composition step is pulled from", "the steps served", 30*time.Second)
select {
case m := <-taken:
var at meshwants.AskTakenEvent
if err := json.Unmarshal(m.Data, &at); err != nil || at.Ask != "validate-9" || at.Node != "n1" || at.At.IsZero() || m.Subject != meshwants.SubjectAskTaken("validate-9") {
t.Fatalf("ask-taken on %s = %+v, %v; want validate-9 taken on n1", m.Subject, at, err)
}
case <-time.After(30 * time.Second):
t.Fatal("no ask-taken within 30 s of the start for the ask put while the assigner was stopped")
}
select {
case e := <-ended:
if e.ID != "validate-9" || e.Outcome != meshwants.StepEndedOutcomeFailed || e.Said != reasonCycle+": a:x → b:y → a:x" {
t.Errorf("the validate step of a take-in making a cycle ended %+v", e)
}
case <-time.After(30 * time.Second):
t.Fatal("the validate ask put while the assigner was stopped said no end")
}
select {
case <-b.seatListRead:
case <-time.After(30 * time.Second):
@@ -306,12 +329,9 @@ func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T
id := fmt.Sprintf("validate-%d", revision)
step, _ := json.Marshal(meshwants.StepRun{Pipeline: fmt.Sprintf("%s.%d", trunkBranch, revision), Step: meshwants.StepRunStepValidate, Module: module,
Branch: trunkBranch, Revision: revision, Commit: commit})
msg := nats.NewMsg(meshwants.AcceptSubjectStep)
msg.Header.Set("x-event-id", id)
msg.Data = step
if err := nc.PublishMsg(msg); err != nil {
t.Fatal(err)
}
var asked meshwants.StepRun
_ = json.Unmarshal(step, &asked)
putStep(b.ctx, t, b.js, id, asked)
if run != nil {
run()
}
@@ -365,10 +385,8 @@ func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T
if e := ask(8, "web", validator); e.Outcome != meshwants.StepEndedOutcomePassed {
t.Errorf("the validate step of a manifest with no problem ended %+v", e)
}
if e := ask(9, "a", nil); e.Outcome != meshwants.StepEndedOutcomeFailed || e.Said != reasonCycle+": a:x → b:y → a:x" {
t.Errorf("the validate step of a take-in making a cycle ended %+v", e)
}
if info, err := worker.Info(b.ctx); err != nil || info.NumPending != 0 {
t.Errorf("a validator is asked for the cycle: %+v %v", info, err)
}
queueEmpty(b.ctx, t, b.js) // every ask acknowledged at its end (§777)
}
@@ -3,15 +3,114 @@ package main
import (
"context"
"encoding/json"
"errors"
"slices"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// fakeMsg is one ask pulled from the seat's work queue, recording how it was settled.
type fakeMsg struct {
data []byte
h nats.Header
mu sync.Mutex
acks, naks, terms, inProgresss int
}
func (m *fakeMsg) Data() []byte { return m.data }
func (m *fakeMsg) Headers() nats.Header { return m.h }
func (m *fakeMsg) count(n *int) error { m.mu.Lock(); *n++; m.mu.Unlock(); return nil }
func (m *fakeMsg) Ack() error { return m.count(&m.acks) }
func (m *fakeMsg) Nak() error { return m.count(&m.naks) }
func (m *fakeMsg) Term() error { return m.count(&m.terms) }
func (m *fakeMsg) InProgress() error { return m.count(&m.inProgresss) }
func (m *fakeMsg) settled() (acks, naks, terms, progress int) {
m.mu.Lock()
defer m.mu.Unlock()
return m.acks, m.naks, m.terms, m.inProgresss
}
// fakeTaken is the state taken: its keys, each with its time to live, and a refusal of TakeOnce when refuse is set.
type fakeTaken struct {
mu sync.Mutex
values map[string][]byte
ttls map[string]time.Duration
refuse error
}
func newFakeTaken() *fakeTaken {
return &fakeTaken{values: map[string][]byte{}, ttls: map[string]time.Duration{}}
}
func (f *fakeTaken) TakeOnce(seat, ask string, bound time.Duration) (bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
if f.refuse != nil {
return false, f.refuse
}
key := seat + "." + ask
if _, ok := f.values[key]; ok {
return false, nil
}
f.values[key], f.ttls[key] = []byte(`{}`), bound
return true, nil
}
func (f *fakeTaken) Get(key string) (*stdio.StateEntry, error) {
f.mu.Lock()
defer f.mu.Unlock()
v, ok := f.values[key]
if !ok {
return nil, nil
}
return &stdio.StateEntry{Key: key, Value: v}, nil
}
func (f *fakeTaken) RenewTTL(key string, value any, ttl time.Duration) (uint64, error) {
raw, err := json.Marshal(value)
f.mu.Lock()
defer f.mu.Unlock()
f.values[key], f.ttls[key] = raw, ttl
return 1, err
}
func (f *fakeTaken) Delete(key string) error {
f.mu.Lock()
defer f.mu.Unlock()
delete(f.values, key)
return nil
}
func (f *fakeTaken) Keys() ([]string, error) {
f.mu.Lock()
defer f.mu.Unlock()
var out []string
for k := range f.values {
out = append(out, k)
}
return out, nil
}
// pullOne hands c one ask pulled from the seat's work queue under id, with a state taken of its own when c has none.
func pullOne(c *compositionSteps, id string, body []byte) *fakeMsg {
if c.taken == nil {
c.taken = newFakeTaken()
}
m := &fakeMsg{data: body, h: nats.Header{}}
if id != "" {
m.h.Set("x-event-id", id)
}
c.pulled(context.Background(), m)
return m
}
// saidEvents keeps what a publisher says.
type saidEvents struct {
mu sync.Mutex
@@ -45,17 +144,20 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
Shared: []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}}}
}}
other, _ := json.Marshal(map[string]any{"pipeline": "main.4.m", "step": "build", "module": "m"})
if c.take(context.Background(), "ask-0", other) {
if m := pullOne(c, "ask-0", other); m.terms != 1 {
t.Fatal("a step ask of another step is taken")
}
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main",
Commit: "c0ffee", Revision: 4, Fingerprint: "fp1", Rerun: true, By: "lead@n1"})
if c.take(context.Background(), "", body) {
if m := pullOne(c, "", body); m.terms != 1 {
t.Fatal("a step ask with no id is taken")
}
if !c.take(context.Background(), "ask-1", body) {
if m := pullOne(c, "ask-1", body); m.terms != 0 {
t.Fatal("the composition step is not taken")
}
if e := <-said.heard; e != meshwants.EventAskTaken+".ask-1" { // said at the pull (H2 §777)
t.Fatalf("said %s first, want ask-taken", e)
}
select {
case <-said.heard:
case <-time.After(5 * time.Second):
@@ -73,6 +175,204 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
len(end.Shared) != 1 || end.Shared[0] != (meshwants.Shared{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}) { // §723, §729
t.Fatalf("step-ended %+v", end)
}
// Design 47 H2 §777, §775 (H-A-381, H-A-401, H-A-404): the asks of the seat's work queue, taken once by id, said ask-taken at
// the pull, in progress while they run, acknowledged once their end is said, and ended by the seat's drop.
t.Run("work queue", queueCases)
}
func queueCases(t *testing.T) {
at := time.Date(2026, 11, 22, 1, 0, 0, 0, time.UTC)
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepValidate, Module: "m", Branch: "main", Revision: 4})
type bed struct {
c *compositionSteps
said *saidEvents
taken *fakeTaken
runs chan string
release chan struct{}
tick chan time.Time
}
newBed := func() *bed {
b := &bed{said: newSaidEvents(), taken: newFakeTaken(), runs: make(chan string, 8), release: make(chan struct{}), tick: make(chan time.Time)}
b.said.heard = make(chan string, 16)
run := func(ctx context.Context, id string, _ meshwants.StepRun) stepOutcome {
b.runs <- id
select {
case <-b.release:
case <-ctx.Done():
}
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
}
b.c = &compositionSteps{node: "n1", say: b.said.say, now: func() time.Time { return at }, logf: t.Logf, taken: b.taken, takenBound: 10 * time.Minute,
validate: run, inProgress: func() time.Duration { return 20 * time.Second },
ticks: func(d time.Duration) (<-chan time.Time, func()) {
if d == 20*time.Second {
return b.tick, func() {}
}
return make(chan time.Time), func() {}
}}
return b
}
ended := func(t *testing.T, b *bed) meshwants.StepEnded {
t.Helper()
for {
select {
case e := <-b.said.heard:
if e == meshwants.EventStepEnded+".n1" {
b.said.mu.Lock()
defer b.said.mu.Unlock()
return b.said.said[e].(meshwants.StepEnded)
}
case <-time.After(5 * time.Second):
t.Fatal("no step-ended is said")
}
}
}
settledAt := func(t *testing.T, m *fakeMsg, acks int) {
t.Helper()
deadline := time.After(5 * time.Second)
for {
if a, _, _, _ := m.settled(); a == acks {
return
}
select {
case <-deadline:
t.Fatalf("the ask is acknowledged %d times, want %d", m.acks, acks)
case <-time.After(time.Millisecond):
}
}
}
t.Run("taken once, said, in progress, acknowledged at its end", func(t *testing.T) {
b := newBed()
m := pullOne(b.c, "ask-1", body)
if id := <-b.runs; id != "ask-1" {
t.Fatalf("ran %s", id)
}
b.said.mu.Lock()
taken, ok := b.said.said[meshwants.EventAskTaken+".ask-1"].(meshwants.AskTakenEvent)
takenID := b.said.ids[meshwants.EventAskTaken+".ask-1"]
b.said.mu.Unlock()
if !ok || taken.Ask != "ask-1" || taken.Node != "n1" || !taken.At.Equal(at) || takenID != "ask-1.taken" || b.taken.ttls["mesh-wants.ask-1"] != 10*time.Minute {
t.Fatalf("ask-taken %+v under %q, taken %v", taken, takenID, b.taken.ttls)
}
b.tick <- at
b.tick <- at
// handed again while it runs: held for the run, which answers both at its one end
again := pullOne(b.c, "ask-1", body)
if a, n, _, p := m.settled(); a != 0 || n != 0 || p < 1 {
t.Fatalf("before its end the ask is acked %d, nakked %d, in progress %d times", a, n, p)
}
close(b.release)
if e := ended(t, b); e.ID != "ask-1" || e.Outcome != meshwants.StepEndedOutcomePassed {
t.Fatalf("step-ended %+v", e)
}
settledAt(t, m, 1)
settledAt(t, again, 1)
select {
case id := <-b.runs:
t.Fatalf("the ask handed again ran again (%s)", id)
default:
}
})
t.Run("taken before is run again, said no ask-taken", func(t *testing.T) {
b := newBed()
_, _ = b.taken.TakeOnce(meshwants.Seat, "ask-2", time.Minute)
close(b.release)
m := pullOne(b.c, "ask-2", body)
<-b.runs
ended(t, b)
settledAt(t, m, 1)
b.said.mu.Lock()
_, said := b.said.said[meshwants.EventAskTaken+".ask-2"]
b.said.mu.Unlock()
if said {
t.Fatal("ask-taken is said again for an ask taken before")
}
})
t.Run("a refused take is handed again", func(t *testing.T) {
b := newBed()
b.taken.refuse = errors.New("no bus")
m := pullOne(b.c, "ask-3", body)
if _, n, _, _ := m.settled(); n != 1 || len(b.runs) != 0 {
t.Fatalf("nakked %d, ran %d", n, len(b.runs))
}
})
t.Run("dropped while it runs", func(t *testing.T) {
b := newBed()
m := pullOne(b.c, "ask-4", body)
<-b.runs
if _, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-4"}); err == nil {
t.Fatal("a drop with no why is taken")
}
a, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-4", Why: "its bound passed"})
if err != nil || a["held"] != true {
t.Fatalf("drop answered %v, %v", a, err)
}
e := ended(t, b)
if e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || e.Said != "the ask was dropped by n1.mesh-pipeline-runner: its bound passed" {
t.Fatalf("step-ended %+v", e)
}
settledAt(t, m, 1)
})
t.Run("dropped before it was pulled", func(t *testing.T) {
b := newBed()
a, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-5", By: "the runner", Why: "its bound passed"})
if err != nil || a["held"] != false || b.taken.ttls["dropped.ask-5"] != 10*time.Minute {
t.Fatalf("drop answered %v, %v; taken %v", a, err, b.taken.ttls)
}
m := pullOne(b.c, "ask-5", body)
e := ended(t, b)
if e.ID != "ask-5" || e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || e.Said != "the ask was dropped by the runner: its bound passed" {
t.Fatalf("step-ended %+v", e)
}
settledAt(t, m, 1)
if len(b.runs) != 0 {
t.Fatal("a dropped ask ran")
}
if v, _ := b.taken.Get("dropped.ask-5"); v != nil {
t.Fatal("the drop's key stays once its ask ended")
}
})
t.Run("the asks tools", func(t *testing.T) {
taken := newFakeTaken()
_, _ = taken.TakeOnce(meshwants.Seat, "ask-6", time.Minute)
_, _ = taken.RenewTTL("dropped.ask-7", askDrop{By: "the runner", Why: "its bound passed", At: at}, time.Minute)
waiting := func(seat string) (stdio.Waiting, error) {
if seat != meshwants.Seat {
t.Errorf("the waiting asks of %s are read", seat)
}
return stdio.Waiting{Asks: []stdio.WaitingAsk{{ID: "ask-8", Module: "mesh-delivery-pipeline", AgeSeconds: 3},
{ID: "ask-6", Module: "mesh-delivery-pipeline", Again: true}, {ID: "ask-9", Module: "mesh-delivery-pipeline", Again: true}}}, nil
}
var asked []string
ask := func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
asked = append(asked, key+" "+string(raw))
if strings.HasSuffix(key, "."+meshwants.VerbDrop) {
return json.RawMessage(`{"held":false}`), nil
}
return json.RawMessage(`{}`), nil
}
tools := map[string]stdio.Tool{}
for _, tool := range asksTools(waiting, taken, ask) {
tools[tool.Name] = tool
}
out, err := tools[asksTool].Run(map[string]any{})
a, _ := out.(map[string]any)
w, _ := a["waiting"].([]map[string]any)
d, _ := a["dropped"].([]map[string]any)
if err != nil || len(w) != 3 || w[0]["id"] != "ask-8" || w[0]["state"] != "waiting" || w[1]["state"] != "held" || w[2]["state"] != "waiting again" || !slices.Equal(a["taken"].([]string), []string{"ask-6"}) || len(d) != 1 ||
d[0]["id"] != "ask-7" || d[0]["by"] != "the runner" || a["total"] != 5 || a["capped"] != false {
t.Fatalf("assigner_asks answered %+v, %v", out, err)
}
if _, err := tools[askDropTool].Run(map[string]any{"ask": "ask-8"}); err == nil || len(asked) != 0 {
t.Fatalf("a drop with no why: %v, asked %v", err, asked)
}
out, err = tools[askDropTool].Run(map[string]any{"ask": "ask-8", "why": "stuck"})
if err != nil || len(asked) != 2 || !strings.HasPrefix(asked[0], "seat:mesh-hand-acts.hand-act ") || !strings.Contains(asked[0], askDropTool+" ask-8") ||
!strings.HasPrefix(asked[1], "seat:mesh-wants.drop ") || !strings.Contains(asked[1], `"id":"ask-8"`) || !strings.Contains(asked[1], `"why":"stuck"`) {
t.Fatalf("assigner_ask_drop answered %v, %v; asked %v", out, err, asked)
}
})
}
// Design 47 G §496, A-262: a running composition step says step-progress.<node> on every tick of its ticker, with the ask, its
@@ -98,9 +398,10 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
}}
// A11/AC1 (design 47 H §752, H-A-88): with no callers in mesh-wants, the progress names the asker the step run carries.
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main", Asker: "mesh-pipeline-runner"})
if !c.take(context.Background(), "ask-1", body) {
if m := pullOne(c, "ask-1", body); m.terms != 0 {
t.Fatal("the composition step is not taken")
}
<-said.heard // ask-taken, at the pull (H2 §777)
progress := meshwants.EventStepProgress + ".n1"
for range 2 {
tick <- at
@@ -149,7 +450,7 @@ func TestASupersededStepLeavesItsRuns(t *testing.T) {
}}
body := func(s meshwants.StepRun) []byte { raw, _ := json.Marshal(s); return raw }
for _, p := range []string{"main.3.a", "main.3.b"} {
if !c.take(context.Background(), "ask-"+p, body(step(p))) {
if m := pullOne(c, "ask-"+p, body(step(p))); m.terms != 0 {
t.Fatal("not taken")
}
}
@@ -177,7 +177,7 @@ func startBed(t *testing.T, leaveOut string) *bed {
continue
}
cfg := jetstream.KeyValueConfig{Bucket: name, History: 5}
if name == plc.PulsesBucket() || name == buckets.CompositionRuns() {
if name == plc.PulsesBucket() || name == buckets.CompositionRuns() || name == buckets.Taken() {
cfg.History, cfg.LimitMarkerTTL = 1, time.Minute // a time to live per key, as the controller makes it (key-ttl)
}
kv, err := js.CreateKeyValue(ctx, cfg)
@@ -206,6 +206,9 @@ func startBed(t *testing.T, leaveOut string) *bed {
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil {
t.Fatal(err)
}
// The seat mesh-wants' work queue and its one worker, as the broker derives them from the seat's definition (design 47 H2
// §777); the assigner pulls its step asks from it.
seatQueue(ctx, t, js)
// The node-engine seat's stream of `running`, kept last per subject, which the assigner hears the health of what runs from (§520).
if _, err := js.CreateStream(ctx, keptStream(nodeengine.StreamRunning, nodeengine.SubjectRunningAll, jetstream.StreamConfig{MaxMsgsPerSubject: 1})); err != nil {
t.Fatal(err)
@@ -1983,3 +1986,61 @@ func serveSeatList(t *testing.T, nc *nats.Conn) <-chan struct{} {
}
return read
}
// seatQueue makes the seat mesh-wants' work queue and its one shared worker from the seat's definition, as the broker derives
// them (design 47 H2 §777).
func seatQueue(ctx context.Context, t *testing.T, js jetstream.JetStream) {
t.Helper()
raw, err := os.ReadFile(filepath.Join("..", "..", "..", "mesh-delivery-contracts", "seats", meshwants.Seat, "seat.json"))
if err != nil {
t.Fatal(err)
}
def, err := seatdef.Parse(raw)
if err != nil {
t.Fatal(err)
}
stream, worker, ok, err := buscompose.SeatQueue(meshwants.Seat, def.Accepts)
if err != nil || !ok {
t.Fatalf("the seat's work queue: %v %v", ok, err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream.Name, Subjects: stream.Subjects, Retention: jetstream.WorkQueuePolicy,
MaxAge: time.Duration(stream.MaxAge) * time.Second, MaxMsgs: int64(stream.MaxMsgs), Duplicates: time.Duration(stream.DuplicatesSeconds) * time.Second}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateOrUpdateConsumer(ctx, stream.Name, jetstream.ConsumerConfig{Durable: worker.Name, FilterSubjects: worker.Filters, AckPolicy: jetstream.AckExplicitPolicy,
AckWait: time.Duration(worker.AckWaitSeconds) * time.Second, MaxDeliver: worker.MaxDeliver, MaxAckPending: worker.MaxAckPending}); err != nil {
t.Fatal(err)
}
}
// putStep puts one step ask on the seat mesh-wants' work queue under id, as the runner puts it (design 47 H2 §777).
func putStep(ctx context.Context, t *testing.T, js jetstream.JetStream, id string, step meshwants.StepRun) {
t.Helper()
msg := nats.NewMsg(meshwants.AcceptSubjectStep)
msg.Header.Set("x-event-id", id)
msg.Header.Set(nats.MsgIdHdr, id)
msg.Data, _ = json.Marshal(step)
if _, err := js.PublishMsg(ctx, msg); err != nil {
t.Fatalf("the ask %s is not put on the seat's work queue: %v", id, err)
}
}
// queueEmpty fails t unless the seat mesh-wants' worker holds no ask, waiting or unacknowledged (design 47 H2 §777).
func queueEmpty(ctx context.Context, t *testing.T, js jetstream.JetStream) {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for {
c, err := js.Consumer(ctx, buscompose.SeatQueueStreamName(meshwants.Seat), buscompose.SeatQueueWorkerName(meshwants.Seat))
if err != nil {
t.Fatal(err)
}
info, err := c.Info(ctx)
if err == nil && info.NumPending == 0 && info.NumAckPending == 0 {
return
}
if time.Now().After(deadline) {
t.Fatalf("the seat's worker holds asks after their ends: %+v %v", info, err)
}
time.Sleep(20 * time.Millisecond)
}
}
@@ -14,8 +14,10 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"io"
"log"
"maps"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
@@ -23,6 +25,7 @@ import (
"os/signal"
"slices"
"strings"
"sync/atomic"
"syscall"
"time"
@@ -89,7 +92,7 @@ func ownBuckets() []string {
return []string{buckets.Asks(), buckets.Differences(),
buckets.Puts(), buckets.Wants(), plc.AssignmentsBucket(), plc.ProposalsBucket(),
plc.CandidatesBucket(), plc.PulsesBucket(), assign.FindingsBucket(), assign.PoliciesBucket(), plc.SeatHoldersBucket(),
engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(), buckets.CompositionRuns()}
engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(), buckets.CompositionRuns(), buckets.Taken()}
}
// engineReadsState is the assigner's state of what each node's engine says it reads (design 47 H-A-104), one key `<node>`
@@ -136,6 +139,7 @@ var keptStates = []announce.State{
{Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211)
{Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354)
{Name: buckets.CompositionRunsState, Readable: true}, // the composition runs, read by the process and its tools (design 47 G §719, A-468c)
{Name: process.TakenState, Readable: true}, // the asks of the seat mesh-wants taken once, and their drops (design 47 H2 §775, §777)
}
// kvSet is every bucket the assigner opened, by name.
@@ -669,8 +673,19 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil {
run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept,
trunk: trunkOf(iv, w.trunkName), newest: iv.newestOf(), watched: iv.watchedOf, assigned: w.assignedOn, assignment: w.assignmentOf, findings: findings, running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf}
// The seat's work queue (design 47 H2 §777): pulled through its one worker, each ask taken once in the state taken.
takenBound, err := askTakenBound()
if err != nil {
return err
}
var ackWait atomic.Int64 // the worker's ack wait, read when it is bound
ackWait.Store(int64(buscompose.SeatQueueDefaultAckWait))
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume,
validate: run.validate, resumeValidate: run.resumeValidate, ended: runs.stepEnded, now: time.Now, logf: logger.Printf}
validate: run.validate, resumeValidate: run.resumeValidate, ended: runs.stepEnded, now: time.Now, logf: logger.Printf,
takenBound: takenBound, inProgress: inProgressOf(seatlist.Definition(gate.Reader, meshwants.Seat), func() time.Duration { return time.Duration(ackWait.Load()) })}
if e, ok := bus.(emitting); ok {
steps.taken = e.State(process.TakenState)
}
move := steps.move(ctx) // the readers a rerun carried (§727, A-470d)
runs.moved.Store(&move)
// The state's watch wakes the steps waiting on a run and hears a bound's expiry; the runs standing at the start are taken up
@@ -685,9 +700,11 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
_, err := meshhandactsclient.Recorder{Ask: bus.Call}.Record(q)
return err
}
stopVerbs, err := e.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), compositionVerbs(runs, steps, record))
verbs := compositionVerbs(runs, steps, record)
maps.Copy(verbs, dropVerb(steps)) // the seat's act drop (§778, H-A-404)
stopVerbs, err := e.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), verbs)
if err != nil {
logger.Printf("the verbs %s and %s are not answered until the assigner starts again: %v", meshwants.VerbCompositions, meshwants.VerbCompositionsDrop, err)
logger.Printf("the verbs %s, %s and %s are not answered until the assigner starts again: %v", meshwants.VerbCompositions, meshwants.VerbCompositionsDrop, meshwants.VerbDrop, err)
} else {
go func() { <-ctx.Done(); stopVerbs() }()
}
@@ -700,10 +717,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
p.ends.left.Store(&left)
}
if err := serveCompositionSteps(ctx, nc, steps); err != nil {
logger.Printf("the composition step is not heard, so no composition step runs until the assigner starts again: %v", err)
} else {
logger.Printf("the composition step is heard on %s", meshwants.AcceptSubjectStep)
if steps.taken == nil {
logger.Printf("the state %s is not bound, so no composition step is pulled until the assigner starts again", process.TakenState)
} else if err := serveStepQueue(ctx, nc, js, steps, func(d time.Duration) { ackWait.Store(int64(d)) }); err != nil {
logger.Printf("the composition step is not pulled, so no composition step runs until the assigner starts again: %v", err)
}
}
if bus != nil {
@@ -0,0 +1,318 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// **The seat mesh-wants takes its asks from a work queue and says ask-taken** (design 47 H2 §777, §775, §778; ADR 0525 §2,
// §6; ADR 0668 §2; H-A-381, H-A-401, H-A-404): the broker derives the seat's stream and its one shared worker from the seat's
// definition; the assigner pulls every ask waiting at once, takes each once by its id in its state taken (key
// <seat>.<ask>, for ask-taken-bound), says ask-taken at the pull, sends in-progress while the step runs and acknowledges an
// ask once it said the ask's end. An ask put while the assigner is down waits on the stream. The seat's act drop ends an ask
// its caller no longer waits for: one running here is cancelled, one not pulled yet is kept in taken as dropped.<ask> for
// ask-taken-bound, and either ends its step could not run naming the drop and who asked it.
// askTakenBoundSetting is the setting the taken key's time to live is (H-A-381 (2)): the holder never receives the
// caller's bound, so the largest caller's, the runner's ask-unanswered-bound, is its default.
const (
askTakenBoundSetting = "ask-taken-bound"
defaultAskTakenBound = 10 * time.Minute
)
// droppedPrefix keys a drop of an ask not pulled yet in the state taken (H-A-404).
const droppedPrefix = "dropped."
// takenStore is the state taken (process.State): an ask taken once by its id, and the drops of asks not pulled yet.
type takenStore interface {
TakeOnce(seat, ask string, bound time.Duration) (bool, error)
Get(key string) (*stdio.StateEntry, error)
RenewTTL(key string, value any, ttl time.Duration) (uint64, error)
Delete(key string) error
}
// queueMsg is what the steps read of a message pulled from the seat's work queue (jetstream.Msg).
type queueMsg interface {
Data() []byte
Headers() nats.Header
Ack() error
Nak() error
Term() error
InProgress() error
}
// askDrop is one drop of an ask: who asked it, why and when; the cause of a running step's end.
type askDrop struct {
By string `json:"by,omitempty"`
Why string `json:"why"`
At time.Time `json:"at"`
}
func (d *askDrop) Error() string {
by := d.By
if by == "" {
by = "its asker"
}
return fmt.Sprintf("the ask was dropped by %s: %s", by, d.Why)
}
// askIDOf is the ask's id a put carries: its event id, else its message id.
func askIDOf(h nats.Header) string {
if id := h.Get("x-event-id"); id != "" {
return id
}
return h.Get(nats.MsgIdHdr)
}
// pulled handles one ask pulled from the seat's work queue: an ask that is no composition or validate step with an id is
// terminated, said; one dropped before it was pulled ends could not run naming the drop; one running here already (handed
// again while it runs) is held for that run, which acknowledges it at its end; any other is taken once by its id (ask-taken
// said the first time) and run, in progress while it runs. A repeat of an id taken before that runs here no longer is run
// again against the runs as they stand (H-A-381 (1), H-A-401): its composition runs are joined, never run twice.
func (c *compositionSteps) pulled(ctx context.Context, m queueMsg) {
id := askIDOf(m.Headers())
var run meshwants.StepRun
err := json.Unmarshal(m.Data(), &run)
how := c.howOf(run.Step, false)
if err != nil || id == "" || how == nil {
c.logf("a step ask on the seat %s's work queue that is no composition or validate step with an id is ended: %v", meshwants.Seat, err)
_ = m.Term()
return
}
if d := c.droppedBefore(id); d != nil {
c.hold(id, m)
c.logf("the ask %s (%s) was dropped before it was pulled: %v", id, run.Pipeline, d)
c.sayEnd(id, run, nil, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: d.Error()}, "")
if err := c.taken.Delete(droppedPrefix + id); err != nil {
c.logf("the drop of the ask %s cannot be deleted from the state %s: %v", id, process.TakenState, err)
}
return
}
if c.running(id) {
c.hold(id, m)
c.logf("the ask %s (%s) was handed again while it runs here: its run answers it", id, run.Pipeline)
return
}
first, err := c.taken.TakeOnce(meshwants.Seat, id, c.takenBound)
if err != nil {
c.logf("the ask %s (%s) is not taken, the state %s refused it: %v; it is handed again", id, run.Pipeline, process.TakenState, err)
_ = m.Nak()
return
}
c.hold(id, m)
if first {
at := c.now().UTC()
if err := c.say(meshwants.EventAskTaken+"."+id, meshwants.AskTakenEvent{Ask: id, Node: c.node, At: at}, id+".taken"); err != nil {
c.logf("saying ask-taken of %s: %v", id, err)
}
} else {
c.logf("the ask %s (%s) was taken before: it is run again against its composition runs as they stand", id, run.Pipeline)
}
go c.say1(ctx, id, run, how, "")
}
// hold keeps m for the step asked under id: acknowledged once its end is said, in progress while it runs.
func (c *compositionSteps) hold(id string, m queueMsg) {
c.mu.Lock()
defer c.mu.Unlock()
if c.held == nil {
c.held = map[string][]queueMsg{}
}
c.held[id] = append(c.held[id], m)
}
// settle acknowledges (ack) or hands again (nak) every message held for id.
func (c *compositionSteps) settle(id string, ack bool) {
c.mu.Lock()
ms := c.held[id]
delete(c.held, id)
c.mu.Unlock()
for _, m := range ms {
if ack {
_ = m.Ack()
} else {
_ = m.Nak()
}
}
}
// inProgressEvery says every message held for id in progress every d, until the returned stop is called.
func (c *compositionSteps) inProgressEvery(id string) (stop func()) {
every := time.Duration(0)
if c.inProgress != nil {
every = c.inProgress()
}
if every <= 0 {
return func() {}
}
ticks := c.ticks
if ticks == nil {
ticks = func(d time.Duration) (<-chan time.Time, func()) { t := time.NewTicker(d); return t.C, t.Stop }
}
tick, stopTicks := ticks(every)
done, ended := make(chan struct{}), make(chan struct{})
go func() {
defer close(ended)
defer stopTicks()
for {
select {
case <-done:
return
case <-tick:
c.mu.Lock()
ms := append([]queueMsg(nil), c.held[id]...)
c.mu.Unlock()
for _, m := range ms {
_ = m.InProgress()
}
}
}
}()
return func() { close(done); <-ended }
}
// droppedBefore is the drop kept for an ask not pulled yet, nil for none.
func (c *compositionSteps) droppedBefore(id string) *askDrop {
if c.taken == nil {
return nil
}
e, err := c.taken.Get(droppedPrefix + id)
if err != nil {
c.logf("whether the ask %s was dropped cannot be read from the state %s: %v; it is taken", id, process.TakenState, err)
return nil
}
if e == nil {
return nil
}
var d askDrop
if json.Unmarshal(e.Value, &d) != nil {
return nil
}
return &d
}
// dropAsk is the seat's act drop (§778, H-A-404): a step running here for the ask is cancelled with the drop as its cause and
// ends could not run naming it; one not running here is kept in taken as dropped.<ask> for ask-taken-bound, so its pull ends
// it the same way. It answers whether the ask was held here.
func (c *compositionSteps) dropAsk(caller string, in meshwants.DropInput) (meshwants.DropAnswer, error) {
id, why := strings.TrimSpace(in.ID), strings.TrimSpace(in.Why)
if id == "" || why == "" {
return nil, errors.New("give the ask's id and why it is dropped")
}
by := strings.TrimSpace(in.By)
if by == "" {
by = caller
}
d := &askDrop{By: by, Why: why, At: c.now().UTC()}
c.mu.Lock()
stop := c.stops[id]
c.mu.Unlock()
if stop != nil {
stop(d)
c.logf("the ask %s running here is dropped: %v", id, d)
return meshwants.DropAnswer{"held": true}, nil
}
if c.taken == nil {
return nil, fmt.Errorf("the state %s is not bound, so the drop of %s cannot be kept", process.TakenState, id)
}
if _, err := c.taken.RenewTTL(droppedPrefix+id, d, c.takenBound); err != nil {
return nil, fmt.Errorf("writing the drop of %s in the state %s: %w", id, process.TakenState, err)
}
c.logf("the ask %s, not running here, is kept dropped until it is pulled: %v", id, d)
return meshwants.DropAnswer{"held": false}, nil
}
// dropVerb is the process's handler of the seat mesh-wants' act drop.
func dropVerb(c *compositionSteps) map[string]process.SeatHandler {
return map[string]process.SeatHandler{
meshwants.VerbDrop: func(caller string, raw json.RawMessage) (any, error) {
var in meshwants.DropInput
if err := json.Unmarshal(raw, &in); err != nil {
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
}
return c.dropAsk(caller, in)
},
}
}
// inProgressOf is the seat's in-progress as its definition names it, once the seat list is read; before, a third of the
// worker's ack wait, which the broker derived from the same definition (60 s and 20 s, §777; H-A-401).
func inProgressOf(def func() (seatdef.Definition, bool), ackWait func() time.Duration) func() time.Duration {
return func() time.Duration {
if d, ok := def(); ok {
if _, worker, ok, err := buscompose.SeatQueue(meshwants.Seat, d.Accepts); err == nil && ok && worker.InProgressSeconds > 0 {
return time.Duration(worker.InProgressSeconds) * time.Second
}
}
return ackWait() / 3
}
}
// serveStepQueue pulls the seat's work queue through its one shared worker, every ask waiting at once (at most the worker's
// max ack pending), once the broker made it: a worker not there yet is said, and bound when the bus says it was created.
func serveStepQueue(ctx context.Context, conn process.Subscriber, js jetstream.JetStream, c *compositionSteps, ackWait func(time.Duration)) error {
stream, worker := buscompose.SeatQueueStreamName(meshwants.Seat), buscompose.SeatQueueWorkerName(meshwants.Seat)
var once sync.Once
bind := func() bool {
cons, err := js.Consumer(ctx, stream, worker)
if err != nil {
return false
}
bound := false
once.Do(func() {
bound = true
info := cons.CachedInfo()
if info != nil && info.Config.AckWait > 0 {
ackWait(info.Config.AckWait)
}
pending := 1000
if info != nil && info.Config.MaxAckPending > 0 {
pending = info.Config.MaxAckPending
}
cc, err := cons.Consume(func(m jetstream.Msg) { c.pulled(ctx, m) }, jetstream.PullMaxMessages(pending))
if err != nil {
c.logf("the seat %s's work queue %s is not pulled: %v", meshwants.Seat, worker, err)
return
}
c.logf("the composition step is pulled from %s on %s", worker, stream)
go func() { <-ctx.Done(); cc.Stop() }()
})
return bound
}
if bind() {
return nil
}
c.logf("the seat %s's work queue %s on %s is not there yet: its asks wait on it, and it is pulled when the bus says it was created", meshwants.Seat, worker, stream)
sub, err := conn.Subscribe("$JS.EVENT.ADVISORY.CONSUMER.CREATED."+stream+"."+worker, func(*nats.Msg) { go bind() })
if err != nil {
return fmt.Errorf("hearing the creation of the seat %s's work queue: %w", meshwants.Seat, err)
}
go func() { <-ctx.Done(); _ = sub.Unsubscribe() }()
if bind() { // made between the first look and the subscription
_ = sub.Unsubscribe()
}
return nil
}
// askTakenBound reads the setting ask-taken-bound (§781).
func askTakenBound() (time.Duration, error) {
d, err := stdio.SettingDuration(askTakenBoundSetting, defaultAskTakenBound)
if err != nil {
return 0, errors.New("the assigner's " + err.Error())
}
return d, nil
}
@@ -22,6 +22,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
"mesh-delivery-assigner/internal/buckets"
@@ -416,6 +417,7 @@ func serveTools() error {
tools = append(tools, whyTools(runtimeStore{})...)
tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...)
tools = append(tools, wantDropTools(runtimeStore{}, stdio.Ask)...)
tools = append(tools, asksTools(stdio.QueueWaiting, stdio.State(process.TakenState), stdio.Ask)...) // the seat's work queue (§777)
return stdio.Serve("", tools)
}
@@ -44,6 +44,10 @@ func Inputs() string { return own(InputsState) }
func CompositionRuns() string { return own(CompositionRunsState) }
func Policies() string { return own(PoliciesState) }
// Taken is the bucket of the state taken (design 47 H2 §775, §777): the asks of the seat mesh-wants taken once, and their
// drops; named as mesh-sdk's process.TakenState.
func Taken() string { return own("taken") }
// AsksHistory is the revisions the bucket of asks keeps per key.
const AsksHistory = 5
@@ -70,7 +70,7 @@ func TestManifest(t *testing.T) {
}
if want := map[string]int{"asks": 5, "differences": 64, "puts": 20, "wants": 20,
"assignments": 64, "placement": 64, "copies": 1, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5, views.MarksState: 1, views.StatusState: 1,
"engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "inputs": 1, "composition-runs": 1}; !reflect.DeepEqual(states, want) {
"engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "inputs": 1, "composition-runs": 1, "taken": 1}; !reflect.DeepEqual(states, want) {
t.Errorf("state = %v, want %v", states, want)
}
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
@@ -93,7 +93,7 @@ func TestManifest(t *testing.T) {
// revision, entry and difference from its revisions kept as objects in the old section, form 1, capped at 512 MiB.
if len(m.Claims) != 2 || m.Claims[0].Name != meshmoduleassigner.Seat || m.Claims[0].At() != catalogue.ScopeMesh ||
!slices.Equal(m.Claims[0].Serves, []string{"assignments", "proposals", "propose", "withdraw", "again", "placement"}) ||
m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input", "compositions", "compositions-drop"}) {
m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input", "compositions", "compositions-drop", "drop"}) {
t.Errorf("the assigner claims %+v, want mesh-module-assigner with its five verbs and mesh-wants with its five", m.Claims)
}
if len(m.Objects) != 1 || m.Objects[0].Name != "assignment-revisions" || m.Objects[0].MaxMiB != 512 || m.Objects[0].Form != 1 {
@@ -105,7 +105,7 @@ func TestManifest(t *testing.T) {
// which lists and asks again what each node's engine reads.
if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_waits", "assigner_inputs", "assigner_inputs_drop", "assigner_compositions", "assigner_compositions_drop", "assignments", "seat-holders", "wants", "asks",
"assigner_read_assignments", "assigner_read_seat_holders", "assigner_read_wants", "assigner_read_findings", "assigner_read_differences", "assigner_read_copies",
"assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_want_drop"}) {
"assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_want_drop", "assigner_asks", "assigner_ask_drop"}) {
t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads, the seat list's two (ADR 0636 §7) assigner_ask_again (ADR 0668 §5) and assigner_why (design 47 G A-439)", m.Tools)
}
if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs {
@@ -172,9 +172,11 @@ func TestManifest(t *testing.T) {
}
file, _ := os.ReadFile("module.json")
if err := json.Unmarshal(file, &raw); err != nil || raw.Config["outside-wait-minutes"].Default != float64(30) || raw.Config["outside-settle-seconds"].Default != float64(90) ||
raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 10 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) ||
raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 13 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) ||
raw.Config["composition-runs-cap-mib"].Default != float64(16) || raw.Config["composition-runs-kept"].Default != "24h" ||
raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) {
raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) ||
// design 47 H2 §775, H-A-381 (2): the taken key's bound and the kept-call pace, runtime settings
raw.Config["ask-taken-bound"].Default != "10m" || raw.Config["kept-call-pace-first"].Default != "1s" || raw.Config["kept-call-pace-max"].Default != "30s" {
t.Errorf("config = %+v (%v), want outside-wait-minutes of 30, outside-settle-seconds of 90 and ask-operator-for of take-off,remove-node,stops (A-425a)", raw.Config, err)
}
for _, f := range []struct{ env, path, key string }{
+28 -3
View File
@@ -27,7 +27,8 @@
"difference",
"input",
"compositions",
"compositions-drop"
"compositions-drop",
"drop"
]
}
],
@@ -129,7 +130,9 @@
"assigner_ask_again",
"assigner_written",
"assigner_why",
"assigner_want_drop"
"assigner_want_drop",
"assigner_asks",
"assigner_ask_drop"
],
"state": [
{ "name": "assignments", "history": 64 },
@@ -152,7 +155,8 @@
{ "name": "waits", "history": 5 },
{ "name": "put-waits", "history": 1 },
{ "name": "inputs", "history": 1, "max-mib": 64 },
{ "name": "composition-runs", "history": 1, "key-ttl": true, "max-mib": 64 }
{ "name": "composition-runs", "history": 1, "key-ttl": true, "max-mib": 64 },
{ "name": "taken", "history": 1, "key-ttl": true, "max-mib": 8 }
],
"config": {
"outside-wait-minutes": {
@@ -226,6 +230,27 @@
"maximum": 1000,
"trusted": false,
"default": 20
},
"ask-taken-bound": {
"type": "string",
"description": "how long the assigner keeps the id of an ask it took from the seat mesh-wants' work queue, so a repeat of it is taken once, and the drop of an ask not pulled yet, a Go duration such as 10m (design 47 H2 §775, §778; H-A-381 (2), H-A-404).",
"trusted": false,
"default": "10m",
"runtime": true
},
"kept-call-pace-first": {
"type": "string",
"description": "the first wait before a kept call met by a transport fault is made again, doubled per call up to kept-call-pace-max, a Go duration (design 47 H2 §775).",
"trusted": false,
"default": "1s",
"runtime": true
},
"kept-call-pace-max": {
"type": "string",
"description": "the most a kept call's wait between calls grows to, a Go duration (design 47 H2 §775).",
"trusted": false,
"default": "30s",
"runtime": true
}
},
"objects": [