From 00b52e3c1ff9c16bd4e88a3e8a62714d57be9a4d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:33:26 +0100 Subject: [PATCH 1/6] BG4, BN4, BI4: the puller says ask-taken, step-progress and the end of any ask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A holder says ask-taken when an ask starts, step-progress every minute while it runs (phase fetch, build or run) and step-ended for an ask outside a pipeline, each on the seat's subject of its node (SayOn); the asker is the last token of the subject the ask came on (Pulled.Subject). Builds: design 47 G §496, §602, §603, §481 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/mesh-delivery-sdk/steprun/phase.go | 7 ++ modules/mesh-delivery-sdk/steprun/pull.go | 103 +++++++++++++++++- modules/mesh-delivery-sdk/steprun/pull_bus.go | 2 +- .../steprun/pull_events_test.go | 93 ++++++++++++++++ modules/mesh-delivery-sdk/steprun/steprun.go | 12 ++ 5 files changed, 215 insertions(+), 2 deletions(-) create mode 100644 modules/mesh-delivery-sdk/steprun/pull_events_test.go diff --git a/modules/mesh-delivery-sdk/steprun/phase.go b/modules/mesh-delivery-sdk/steprun/phase.go index a4e878a7..72d38465 100644 --- a/modules/mesh-delivery-sdk/steprun/phase.go +++ b/modules/mesh-delivery-sdk/steprun/phase.go @@ -65,3 +65,10 @@ func (r *phases) end() sdkstep.Step { r.ended = true return r.step } + +// phase is the phase the ask is in now, "" before its first. +func (r *phases) phase() string { + r.mu.Lock() + defer r.mu.Unlock() + return r.step.Phase +} diff --git a/modules/mesh-delivery-sdk/steprun/pull.go b/modules/mesh-delivery-sdk/steprun/pull.go index c3599866..26677b54 100644 --- a/modules/mesh-delivery-sdk/steprun/pull.go +++ b/modules/mesh-delivery-sdk/steprun/pull.go @@ -27,6 +27,9 @@ type Pulled struct { // Kind is the worker it came from, filled by the puller. Kind string Data []byte + // Subject is the subject the ask was published on: its last token is the asker the bus proved (design 47 G §481); empty + // for a source that has none, and for main's form, which names none. + Subject string // Delivered is how many times the bus has handed it over, this time included. Delivered uint64 // Ack ends it: it is never delivered again. InProgress tells the bus it is still being worked, restarting its @@ -53,6 +56,8 @@ var ( // DefaultHeartbeat is how often a running ask tells the bus it is still being worked; under the seat worker's // acknowledgement wait of 60 s (mesh-controller internal/broker HolderConsumerFor). DefaultHeartbeat = 20 * time.Second + // DefaultProgressEvery is how often a running ask says step-progress (design 47 G §496: once a minute). + DefaultProgressEvery = time.Minute ) // Puller pulls a build seat's asks and runs them. @@ -72,7 +77,12 @@ type Puller struct { // sdkstep.AgainAfter (a minute, then four). A test's own is short. Delay func(run int) time.Duration Heartbeat time.Duration - Logf func(format string, args ...any) + // Say publishes one event of the seat on this node's subject (SayOn): ask-taken when an ask starts, step-progress once + // every ProgressEvery while it runs, step-ended when an ask outside a pipeline ends (design 47 G §496, §602). Nil says + // none, a test's. ProgressEvery defaults to DefaultProgressEvery. + Say func(event string, body any, id string) error + ProgressEvery time.Duration + Logf func(format string, args ...any) // Load reads one key of the state the node's pause is kept in (nil and no error when it is absent). When set, Pull // reads the node's pause key (sdkstep.PauseKey) before it pulls, and Pause and Resume write it with PausePut, so a // pause outlives a restart. @@ -665,6 +675,7 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) { p.running[ask.ID] = RunningAsk{ID: ask.ID, Kind: ask.Kind, Module: module, Started: step.Started} p.mu.Unlock() p.logf("the %s ask %s starts on %s: %s", ask.Kind, ask.ID, p.Node, step.What) + p.say("ask-taken", askTaken{Ask: ask.ID, Asker: askerOf(msg, ask), Node: p.Node, At: now().UTC()}, "ask-taken."+ask.ID) go p.run(ctx, msg, ask, key, step) } @@ -696,6 +707,25 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key } }() ph := &phases{put: p.Put, key: key, step: step, logf: p.Logf, prefix: fmt.Sprintf("the %s ask %s on %s", ask.Kind, ask.ID, p.Node)} + if p.Say != nil { + every := p.ProgressEvery + if every <= 0 { + every = DefaultProgressEvery + } + go func() { + t := time.NewTicker(every) + defer t.Stop() + for { + select { + case <-done: + return + case <-t.C: + p.say("step-progress", stepProgress{Ask: ask.ID, Asker: askerOf(msg, ask), Phase: progressPhase(ask, ph.phase()), At: now().UTC()}, + fmt.Sprintf("step-progress.%s.%d", ask.ID, now().UnixNano())) + } + } + }() + } runCtx := p.cancellable(ctx, ask.ID) result, err := protect(runCtx, func(ctx context.Context) (any, error) { return p.Run(context.WithValue(ctx, phaseKey{}, ph), ask) @@ -713,6 +743,7 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key var dec *Declining if errors.As(err, &dec) && ctx.Err() == nil { p.declineRun(msg, ask, key, step, dec) + p.sayEnded(msg, ask, sdkstep.OutcomeCouldNotRun, dec.Text) return } if ctx.Err() != nil { @@ -750,6 +781,9 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key if len(outcome) > 300 { outcome = outcome[:300] + "..." } + p.sayEnded(msg, ask, sdkstep.OutcomeFailed, err.Error()) + } else { + p.sayEnded(msg, ask, sdkstep.OutcomePassed, "") } p.logf("the %s ask %s ended on %s after %s, in its phase %q; %s", ask.Kind, ask.ID, p.Node, now().Sub(step.Started).Round(time.Second), step.Phase, outcome) @@ -884,3 +918,70 @@ func askRead(ask sdkstep.StepAsk) sdkstep.Ask { out, _ := sdkstep.AskAs[sdkstep.Ask](ask) return out } + +// askTaken and stepProgress are the bodies of the seat's events ask-taken and step-progress (design 47 G §602, §496). +type askTaken struct { + Ask string `json:"ask"` + Asker string `json:"asker"` + Node string `json:"node"` + At time.Time `json:"at"` +} + +type stepProgress struct { + Ask string `json:"ask"` + Asker string `json:"asker"` + Phase string `json:"phase"` + At time.Time `json:"at"` +} + +// askerOf is the asker the bus proved: the last token of the subject the ask came on (design 47 G §481), the ask's own +// field only for a source that gives no subject and for main's form. +func askerOf(msg *Pulled, ask sdkstep.StepAsk) string { + if _, _, asker, err := sdkstep.ParseAskSubjectBy(msg.Subject); err == nil && asker != "" { + return asker + } + return ask.Asker +} + +// progressPhase is the phase step-progress says (fetch, build, run or report): fetch until the ask has entered a phase of +// its own or while it names the fetch, then the ask's kind (build for a build, run for any other). +func progressPhase(ask sdkstep.StepAsk, phase string) string { + switch { + case phase == "" || strings.HasPrefix(phase, "fetch"): + return "fetch" + case ask.Build != nil: + return "build" + } + return "run" +} + +// say publishes one event of the seat; a failure is logged and never fails the ask. +func (p *Puller) say(event string, body any, id string) { + if p.Say == nil { + return + } + if err := p.Say(event, body, id); err != nil { + p.logf("the event %s of the ask %s could not be said on %s: %v", event, id, p.Node, err) + } +} + +// sayEnded says step-ended for an ask outside a pipeline, whatever its asker (design 47 G §602): a pipeline step's end is +// its run's own (AskSteps.SayStep). +func (p *Puller) sayEnded(msg *Pulled, ask sdkstep.StepAsk, outcome, said string) { + if p.Say == nil || ask.Step != nil { + return + } + if len(said) > 300 { + said = said[:300] + "..." + } + if said == "" && outcome != sdkstep.OutcomePassed { + said = "the ask did not pass" + } + end := sdkstep.StepEnded{ID: ask.ID, Asker: askerOf(msg, ask), Ask: ask.ID, Node: p.Node, Outcome: outcome, Said: said, At: now().UTC()} + body, err := sdkstep.EncodeStepEnded(end) + if err != nil { + p.logf("the end of the ask %s cannot be said on %s: %v", ask.ID, p.Node, err) + return + } + p.say("step-ended", json.RawMessage(body), "step-ended."+ask.ID) +} diff --git a/modules/mesh-delivery-sdk/steprun/pull_bus.go b/modules/mesh-delivery-sdk/steprun/pull_bus.go index efedb295..e261a1d0 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_bus.go +++ b/modules/mesh-delivery-sdk/steprun/pull_bus.go @@ -69,7 +69,7 @@ func (s *JetStreamAsks) Next(ctx context.Context, kind string, wait time.Duratio if meta, err := msg.Metadata(); err == nil && meta != nil { delivered = meta.NumDelivered } - return &Pulled{Data: append([]byte(nil), msg.Data()...), Delivered: delivered, + return &Pulled{Data: append([]byte(nil), msg.Data()...), Subject: msg.Subject(), Delivered: delivered, Ack: msg.Ack, InProgress: msg.InProgress, Nak: msg.Nak, Term: msg.Term, NakAfter: func(delay time.Duration) error { return msg.NakWithDelay(delay) }}, nil case <-ctx.Done(): diff --git a/modules/mesh-delivery-sdk/steprun/pull_events_test.go b/modules/mesh-delivery-sdk/steprun/pull_events_test.go new file mode 100644 index 00000000..466cc951 --- /dev/null +++ b/modules/mesh-delivery-sdk/steprun/pull_events_test.go @@ -0,0 +1,93 @@ +package steprun + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + "time" + + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" +) + +type said struct { + event, id string + body map[string]any +} + +type sayer struct { + mu sync.Mutex + got []said +} + +func (s *sayer) say(event string, body any, id string) error { + raw, _ := json.Marshal(body) + var m map[string]any + _ = json.Unmarshal(raw, &m) + s.mu.Lock() + s.got = append(s.got, said{event, id, m}) + s.mu.Unlock() + return nil +} + +func (s *sayer) of(event string) []said { + s.mu.Lock() + defer s.mu.Unlock() + var out []said + for _, g := range s.got { + if g.event == event { + out = append(out, g) + } + } + return out +} + +// Design 47 G §496, §602 (BG4, BN4, BI4): a holder says ask-taken when an ask starts, step-progress while it runs and +// step-ended when an ask outside a pipeline ends, whatever its asker. +func TestPullSaysAskTakenProgressAndEnd(t *testing.T) { + for _, c := range []struct { + name string + runErr error + outcome string + }{{"passed", nil, sdkstep.OutcomePassed}, {"failed", errors.New("the build broke"), sdkstep.OutcomeFailed}} { + t.Run(c.name, func(t *testing.T) { + src, st, sy := NewFakeAsks(), newSteps(), &sayer{} + a := testAsk(sdkstep.AskDelivery) + _ = src.PublishAsk(a) + release, entered := make(chan struct{}), make(chan struct{}) + p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, + PullWait: 10 * time.Millisecond, Heartbeat: time.Hour, Say: sy.say, ProgressEvery: 5 * time.Millisecond, + Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) { + Phase(ctx, "build m") + close(entered) + <-release + return nil, c.runErr + }} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Pull(ctx) + <-entered + waitFor(t, "a step-progress in the build", func() bool { + pr := sy.of("step-progress") + return len(pr) > 0 && pr[len(pr)-1].body["phase"] == "build" + }) + close(release) + waitFor(t, "step-ended", func() bool { return len(sy.of("step-ended")) == 1 }) + taken := sy.of("ask-taken") + if len(taken) != 1 || taken[0].body["ask"] != a.ID || taken[0].body["node"] != "shanks" || taken[0].id != "ask-taken."+a.ID { + t.Fatalf("ask-taken = %+v, want one for %s on shanks", taken, a.ID) + } + if pr := sy.of("step-progress")[len(sy.of("step-progress"))-1].body; pr["ask"] != a.ID || pr["phase"] != "build" || pr["asker"] != a.Asker { + t.Fatalf("step-progress = %v, want the ask, its asker and the phase build", pr) + } + end := sy.of("step-ended")[0].body + if end["outcome"] != c.outcome || end["ask"] != a.ID || end["asker"] != a.Asker || end["node"] != "shanks" || end["at"] == nil { + t.Fatalf("step-ended = %v, want outcome %s of ask %s", end, c.outcome, a.ID) + } + if c.runErr != nil && end["said"] != "the build broke" { + t.Fatalf("step-ended said %v, want why", end["said"]) + } + }) + } +} diff --git a/modules/mesh-delivery-sdk/steprun/steprun.go b/modules/mesh-delivery-sdk/steprun/steprun.go index c8a44c0a..7cd29e6e 100644 --- a/modules/mesh-delivery-sdk/steprun/steprun.go +++ b/modules/mesh-delivery-sdk/steprun/steprun.go @@ -250,3 +250,15 @@ func SayStepOn(language, node string, publish func(subject string, body any, id return publish(subject, json.RawMessage(body), "step-ended."+end.ID) } } + +// SayOn is a holder's Puller.Say: an event of the language's build seat published on that seat's subject of node with +// publish (stdio.SeatPublish), `mesh.seat.node-build-.event..`. +func SayOn(language, node string, publish func(subject string, body any, id string) error) func(event string, body any, id string) error { + return func(event string, body any, id string) error { + seat, err := sdkstep.BuildSeat(language) + if err != nil { + return err + } + return publish("mesh.seat."+seat+".event."+event+"."+node, body, id) + } +} From 7033d5e80d1f13f5473b6b3cfd9c7d9b13ceddbf Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:42:00 +0100 Subject: [PATCH 2/6] BG2, BG3: the puller refuses a kind its asker may not ask and pulls the named worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An ask's asker is the last token of its accept subject; the holder checks the kind against the callers the build seat's definition lists (Puller.Callers, CallersFromDefinitions), refuses an ask of another kind answering error naming the kind and the asker and ending it failed, takes only a build and a run on main's subject, and gives back an ask the seat list does not judge yet. A second worker per queue (AskNamedWorker, filtered to accept..*) is pulled after main's; the holder says `named worker not bound` while its membership lists none. Builds: design 47 G §543, §544, §481, A-163, A-170 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/mesh-delivery-sdk/buildstep/ask.go | 10 ++ modules/mesh-delivery-sdk/steprun/askers.go | 149 ++++++++++++++++++ .../mesh-delivery-sdk/steprun/askers_test.go | 86 ++++++++++ modules/mesh-delivery-sdk/steprun/pull.go | 12 +- modules/mesh-delivery-sdk/steprun/pull_bus.go | 78 +++++++-- .../mesh-delivery-sdk/steprun/pull_fake.go | 20 ++- .../steprun/pull_integration_test.go | 44 +++--- .../mesh-delivery-sdk/steprun/pull_seat.go | 52 +++++- .../steprun/pull_seat_test.go | 35 +++- 9 files changed, 449 insertions(+), 37 deletions(-) create mode 100644 modules/mesh-delivery-sdk/steprun/askers.go create mode 100644 modules/mesh-delivery-sdk/steprun/askers_test.go diff --git a/modules/mesh-delivery-sdk/buildstep/ask.go b/modules/mesh-delivery-sdk/buildstep/ask.go index 64439a89..e78e5c60 100644 --- a/modules/mesh-delivery-sdk/buildstep/ask.go +++ b/modules/mesh-delivery-sdk/buildstep/ask.go @@ -203,6 +203,16 @@ func AskWorker(language, kind string) (string, error) { return stream + "_" + strings.ToUpper(kind) + "_worker", nil } +// AskNamedWorker is the pull worker of a queue's named asks, `SEAT___named_worker`, filtered to +// `mesh.seat..accept..*`, beside main's worker AskWorker, filtered to main's subject alone (design 47 G §544). +func AskNamedWorker(language, kind string) (string, error) { + worker, err := AskWorker(language, kind) + if err != nil { + return "", err + } + return strings.TrimSuffix(worker, "_worker") + "_named_worker", nil +} + // AskStepKey is the key of the step of the ask askID on node. func AskStepKey(node, askID string) (string, error) { if err := CheckAskID(askID); err != nil { diff --git a/modules/mesh-delivery-sdk/steprun/askers.go b/modules/mesh-delivery-sdk/steprun/askers.go new file mode 100644 index 00000000..3023ad38 --- /dev/null +++ b/modules/mesh-delivery-sdk/steprun/askers.go @@ -0,0 +1,149 @@ +package steprun + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + "sync/atomic" + + buildrun "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildrun" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/seatdef" + "git.novox.be/novox/mesh-sdk/go/seatlist" +) + +// **A holder refuses a kind its asker may not ask** (design 47 G §543, A-163, A-170). An ask rides +// `mesh.seat..accept..`, the asker the asking seat's own name, and its kind stays in its body. The +// holder takes the asker from the subject and checks the kind against the callers the seat's definition lists for it (read +// from the seat list); on main's subject, which names no asker, it takes only a build and a run. + +// KindOf is the word the seat definition gives an ask's kind: run for a run or a compose, else the ask's own kind. +func KindOf(ask sdkstep.StepAsk) string { + if ask.Run != nil || ask.Compose != nil { + return "run" + } + return ask.Kind +} + +// CallersFromDefinitions is a Puller.Callers: the seats the definition of language's build seat lists as callers of an +// accepted kind, from the seat list's definitions (seatlist.Reader.Definitions). found is false when the definition or the +// kind is not held yet. A caller entry is `seat:`. +func CallersFromDefinitions(language string, definitions func() map[string]seatdef.Definition) func(kind string) (callers []string, found bool) { + return func(kind string) ([]string, bool) { + seat, err := sdkstep.BuildSeat(language) + if err != nil { + return nil, false + } + def, ok := definitions()[seat] + if !ok { + return nil, false + } + var out []string + found := false + for _, m := range def.Accepts { + if m.Name != kind { + continue + } + found = true + if m.Callers != nil { + for _, e := range m.Callers.List { + if name, ok := strings.CutPrefix(e, "seat:"); ok { + out = append(out, name) + } + } + } + } + return out, found + } +} + +// askerRefusal is why an ask must not be taken from its subject's asker: "" when it may be, and judged false when the seat +// list does not say yet (the ask is given back, not refused). +func (p *Puller) askerRefusal(msg *Pulled, ask sdkstep.StepAsk) (why string, judged bool) { + if msg.Subject == "" { + return "", true // a source that gives no subject (a test's) + } + _, _, asker, err := sdkstep.ParseAskSubjectBy(msg.Subject) + if err != nil { + return fmt.Sprintf("the ask came on %s, which is no accept subject of this seat", msg.Subject), true + } + kind := KindOf(ask) + if asker == "" { + if (ask.Build != nil && ask.Kind == sdkstep.AskDelivery) || kind == "run" { + return "", true + } + return fmt.Sprintf("the kind %s is not taken on main's subject, which takes only a build and a run (asker: none named)", kind), true + } + if p.Callers == nil { + return "", true + } + callers, found := p.Callers(kind) + if !found { + return "", false + } + if !slices.Contains(callers, asker) { + return fmt.Sprintf("the kind %s is not one %s may ask: its callers are %s", kind, asker, strings.Join(callers, ", ")), true + } + return "", true +} + +// refuseAsker ends an ask whose asker may not ask its kind: its step key says failed with the reason, naming the kind and the +// asker, step-ended says failed, and the message is dropped. +func (p *Puller) refuseAsker(msg *Pulled, ask sdkstep.StepAsk, why string) { + key, err := sdkstep.AskStepKey(p.Node, ask.ID) + if err == nil { + at := now().UTC() + step := sdkstep.Step{ID: ask.ID, Tool: ask.Kind + " ask", Node: p.Node, State: sdkstep.StepFailed, Started: at, Ended: at, + Reason: "error: " + why, Owner: buildrun.Owner, Run: int(msg.Delivered)} + err = writeEnd(p.Put, key, step) + } + if err != nil { + p.logf("the ask %s of the wrong asker cannot be answered on %s, so it is given back: %v", ask.ID, p.Node, err) + _ = msg.Nak() + return + } + p.logf("the ask %s is refused on %s: %s", ask.ID, p.Node, why) + if ask.Step == nil { + p.sayEnded(msg, ask, sdkstep.OutcomeFailed, why) + } else if end, err := sdkstep.EndOf(ask, p.Node, sdkstep.OutcomeFailed); err == nil { + // A pipeline step's end names its pipeline, step, commit and fingerprint (its own form, §602). + end.Said, end.At, end.Asker = why, now().UTC(), askerOf(msg, ask) + if body, err := sdkstep.EncodeStepEnded(end); err == nil { + p.say("step-ended", json.RawMessage(body), "step-ended."+ask.ID) + } + } + _ = msg.Term() +} + +// giveBack hands an ask back to be judged again a little later, once the seat list says more. +func (p *Puller) giveBack(msg *Pulled, ask sdkstep.StepAsk) { + p.logf("the ask %s is given back on %s: the seat list holds no definition of its kind yet", ask.ID, p.Node) + if msg.NakAfter != nil && msg.NakAfter(DefaultPullWait) == nil { + return + } + _ = msg.Nak() +} + +// SeatListCallers is a Puller.Callers that follows the seat list: it starts the reader once the runtime serves +// (seatlist.Start through the runtime's own way in) and answers from its definitions, found false until it has read them. +// A reader that cannot start is said and the asks are given back until it can (design 47 G §543, A-163). +func SeatListCallers(language string, logf func(format string, args ...any)) func(kind string) ([]string, bool) { + var reader atomic.Pointer[seatlist.Reader] + go func() { + <-stdio.Served() + r, err := seatlist.Start(seatlist.Config{Bus: seatlist.StdioBus(), Bootstrap: seatlist.Bootstrapped()}) + if err != nil && logf != nil { + logf("following the seat list: %v", err) + } + reader.Store(r) + }() + return func(kind string) ([]string, bool) { + r := reader.Load() + if r == nil { + return nil, false + } + return CallersFromDefinitions(language, r.Definitions)(kind) + } +} diff --git a/modules/mesh-delivery-sdk/steprun/askers_test.go b/modules/mesh-delivery-sdk/steprun/askers_test.go new file mode 100644 index 00000000..aa5ace6e --- /dev/null +++ b/modules/mesh-delivery-sdk/steprun/askers_test.go @@ -0,0 +1,86 @@ +package steprun + +import ( + "context" + "strings" + "testing" + "time" + + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +func callersFor(table map[string][]string) func(string) ([]string, bool) { + return func(kind string) ([]string, bool) { c, ok := table[kind]; return c, ok } +} + +// Design 47 G §543 (BG2, BG3, A-163, A-170): an ask whose kind its asker, the last token of its subject, is no caller of is +// answered error naming the kind and the asker and ended failed; a step on main's subject is refused as well; one the +// seat list does not judge yet is given back. +func TestPullRefusesAKindItsAskerMayNotAsk(t *testing.T) { + step := testAsk(sdkstep.AskStep) + step.Check = nil + step.Step = &sdkstep.StepRun{Pipeline: "p", Step: sdkstep.StepUnit, Module: "m", Artifact: "m:a", Commit: "0123456789abcdef0123456789abcdef01234567", + Fingerprint: "fp", Fetch: sdkstep.Fetch{Repository: "novox/x", Commit: "0123456789abcdef0123456789abcdef01234567"}} + build := testAsk(sdkstep.AskDelivery) + table := map[string][]string{"step": {"mesh-pipeline-runner"}, "delivery": {"mesh-builds"}} + for _, c := range []struct { + name, subject string + ask sdkstep.StepAsk + callers func(string) ([]string, bool) + want string // "" runs it, "refused" refuses it, "back" gives it back + }{ + {"the indexer's step", "mesh.seat.node-build-go.accept.check.mesh-catalog", step, callersFor(table), "refused"}, + {"the runner's step", "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner", step, callersFor(table), ""}, + {"a step on main's subject", "mesh.seat.node-build-go.accept.check", step, callersFor(table), "refused"}, + {"a build on main's subject", "mesh.seat.node-build-go.accept.delivery", build, callersFor(table), ""}, + {"a kind the seat list does not hold", "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner", step, callersFor(nil), "back"}, + } { + t.Run(c.name, func(t *testing.T) { + src, st, sy := NewFakeAsks(), newSteps(), &sayer{} + raw, _ := sdkstep.EncodeStepAsk(c.ask) + src.PublishWithSubject(c.ask.Kind, raw, c.subject) + ran := make(chan struct{}, 1) + p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, Callers: c.callers, + PullWait: 5 * time.Millisecond, Heartbeat: time.Hour, Say: sy.say, ProgressEvery: time.Hour, + Run: func(context.Context, sdkstep.StepAsk) (any, error) { ran <- struct{}{}; return nil, nil }} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Pull(ctx) + switch c.want { + case "": + <-ran + case "refused": + waitFor(t, "the ask ended", func() bool { _, termed, _ := src.Fates(); return len(termed) == 1 }) + key, _ := sdkstep.AskStepKey("shanks", c.ask.ID) + s, _ := st.get(key) + asker := c.subject[strings.LastIndex(c.subject, ".")+1:] + if s.State != sdkstep.StepFailed || !strings.Contains(s.Reason, KindOf(c.ask)) || (strings.Contains(c.subject, "mesh-catalog") && !strings.Contains(s.Reason, asker)) { + t.Fatalf("the step = %+v, want failed naming the kind %s and the asker", s, KindOf(c.ask)) + } + if end := sy.of("step-ended"); len(end) != 1 || end[0].body["outcome"] != sdkstep.OutcomeFailed { + t.Fatalf("step-ended = %+v, want failed", end) + } + select { + case <-ran: + t.Fatal("a refused ask ran") + default: + } + case "back": + waitFor(t, "the ask given back", func() bool { src.mu.Lock(); defer src.mu.Unlock(); return len(src.Delayed) > 0 }) + } + }) + } +} + +func TestCallersFromDefinitionsReadsTheSeatsCallers(t *testing.T) { + def := seatdef.Definition{Seat: "node-build-go", Accepts: []seatdef.Message{ + {Name: "step", Callers: &seatdef.Callers{List: []string{"seat:mesh-pipeline-runner"}}}}} + callers := CallersFromDefinitions(sdkstep.AskGo, func() map[string]seatdef.Definition { return map[string]seatdef.Definition{"node-build-go": def} }) + if got, ok := callers("step"); !ok || len(got) != 1 || got[0] != "mesh-pipeline-runner" { + t.Fatalf("callers of step = %v, %v", got, ok) + } + if _, ok := callers("run"); ok { + t.Fatal("a kind the definition does not list is found") + } +} diff --git a/modules/mesh-delivery-sdk/steprun/pull.go b/modules/mesh-delivery-sdk/steprun/pull.go index 26677b54..77a772d6 100644 --- a/modules/mesh-delivery-sdk/steprun/pull.go +++ b/modules/mesh-delivery-sdk/steprun/pull.go @@ -80,7 +80,10 @@ type Puller struct { // Say publishes one event of the seat on this node's subject (SayOn): ask-taken when an ask starts, step-progress once // every ProgressEvery while it runs, step-ended when an ask outside a pipeline ends (design 47 G §496, §602). Nil says // none, a test's. ProgressEvery defaults to DefaultProgressEvery. - Say func(event string, body any, id string) error + Say func(event string, body any, id string) error + // Callers says which seats the build seat's definition lists as callers of an accepted kind (CallersFromDefinitions); + // found is false while the seat list holds no such kind. Nil checks no named asker (design 47 G §543, A-163). + Callers func(kind string) (callers []string, found bool) ProgressEvery time.Duration Logf func(format string, args ...any) // Load reads one key of the state the node's pause is kept in (nil and no error when it is absent). When set, Pull @@ -640,6 +643,13 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) { p.refuseUnread(msg, err) return } + if why, judged := p.askerRefusal(msg, ask); !judged { + p.giveBack(msg, ask) + return + } else if why != "" { + p.refuseAsker(msg, ask, why) + return + } if d := p.droppedAsk(ask.ID); d != nil { p.refuseDropped(msg, ask, d) return diff --git a/modules/mesh-delivery-sdk/steprun/pull_bus.go b/modules/mesh-delivery-sdk/steprun/pull_bus.go index e261a1d0..5669ec4b 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_bus.go +++ b/modules/mesh-delivery-sdk/steprun/pull_bus.go @@ -16,14 +16,19 @@ type JetStreamAsks struct { JS jetstream.JetStream Language string - mu sync.Mutex - cons map[string]jetstream.Consumer + mu sync.Mutex + cons map[string]jetstream.Consumer + missing map[string]bool } -func (s *JetStreamAsks) consumer(ctx context.Context, kind string) (jetstream.Consumer, error) { +func (s *JetStreamAsks) consumer(ctx context.Context, kind string, named bool) (jetstream.Consumer, error) { s.mu.Lock() defer s.mu.Unlock() - if c, ok := s.cons[kind]; ok { + cacheKey := kind + if named { + cacheKey = kind + "/named" + } + if c, ok := s.cons[cacheKey]; ok { return c, nil } stream, err := sdkstep.AskStream(s.Language) @@ -31,6 +36,9 @@ func (s *JetStreamAsks) consumer(ctx context.Context, kind string) (jetstream.Co return nil, err } worker, err := sdkstep.AskWorker(s.Language, kind) + if named { + worker, err = sdkstep.AskNamedWorker(s.Language, kind) + } if err != nil { return nil, err } @@ -41,17 +49,60 @@ func (s *JetStreamAsks) consumer(ctx context.Context, kind string) (jetstream.Co if s.cons == nil { s.cons = map[string]jetstream.Consumer{} } - s.cons[kind] = c + s.cons[cacheKey] = c return c, nil } -// Next is AskSource's: a fetch of one with no wait when wait is 0, else one that waits at most wait. +// Next is AskSource's: main's worker first, then the queue's named worker once it is bound (design 47 G §544), with no +// wait when wait is 0, else waiting at most wait between the two. A named worker not bound is only main's alone. func (s *JetStreamAsks) Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error) { - c, err := s.consumer(ctx, kind) - if err != nil { - return nil, err + var half time.Duration + if wait > 0 { + half = wait / 2 } + for i, step := range []struct { + named bool + wait time.Duration + }{{false, 0}, {true, 0}, {false, half}, {true, wait - half}} { + if step.named && s.unbound(kind) { + continue + } + if i >= 2 && wait <= 0 { + break // the two waiting pulls are for a wait + } + c, err := s.consumer(ctx, kind, step.named) + if err != nil { + if step.named { + s.markUnbound(kind) + continue + } + return nil, err + } + if msg, err := s.fetch(ctx, c, step.wait); err != nil || msg != nil { + return msg, err + } + } + return nil, nil +} + +func (s *JetStreamAsks) unbound(kind string) bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.missing[kind] +} + +func (s *JetStreamAsks) markUnbound(kind string) { + s.mu.Lock() + if s.missing == nil { + s.missing = map[string]bool{} + } + s.missing[kind] = true + s.mu.Unlock() +} + +func (s *JetStreamAsks) fetch(ctx context.Context, c jetstream.Consumer, wait time.Duration) (*Pulled, error) { var batch jetstream.MessageBatch + var err error if wait <= 0 { batch, err = c.FetchNoWait(1) } else { @@ -93,11 +144,16 @@ func DeclareAskWorkers(ctx context.Context, js jetstream.JetStream, language str for _, kind := range sdkstep.AskKinds { worker, _ := sdkstep.AskWorker(language, kind) subject, _ := sdkstep.AskSubject(language, kind) - // Both forms of the accept subject: main's and the asker's, `.` (design 47 G §600). - if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: worker, FilterSubjects: []string{subject, subject + ".*"}, + // Two workers whose filters do not overlap: main's subject and the asker's, `.` (design 47 G §544). + if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: worker, FilterSubjects: []string{subject}, AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: 5}); err != nil { return fmt.Errorf("the worker %s: %w", worker, err) } + named, _ := sdkstep.AskNamedWorker(language, kind) + if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: named, FilterSubjects: []string{subject + ".*"}, + AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: 5}); err != nil { + return fmt.Errorf("the worker %s: %w", named, err) + } } return nil } diff --git a/modules/mesh-delivery-sdk/steprun/pull_fake.go b/modules/mesh-delivery-sdk/steprun/pull_fake.go index 574605dc..05e648e3 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_fake.go +++ b/modules/mesh-delivery-sdk/steprun/pull_fake.go @@ -13,7 +13,9 @@ import ( type FakeAsks struct { mu sync.Mutex waiting map[string][][]byte - added chan struct{} + // subjects are the subjects asks were published with, by their data. + subjects map[string]string + added chan struct{} // Acked, Termed and Naked are the asks' ids or data by fate, InProgress how many heartbeats were given. Acked, Termed, Naked []string // Delayed are the asks handed back to be given again after a delay (NakAfter), and Delays that delay of each. @@ -40,6 +42,17 @@ func (f *FakeAsks) Publish(kind string, data []byte) { } } +// PublishWithSubject puts an encoded ask on kind's worker as the bus would hand it over from subject (design 47 G §481). +func (f *FakeAsks) PublishWithSubject(kind string, data []byte, subject string) { + f.mu.Lock() + if f.subjects == nil { + f.subjects = map[string]string{} + } + f.subjects[string(data)] = subject + f.mu.Unlock() + f.Publish(kind, data) +} + // PublishAsk encodes a and puts it on its kind's worker. func (f *FakeAsks) PublishAsk(a sdkstep.StepAsk) error { raw, err := sdkstep.EncodeStepAsk(a) @@ -107,7 +120,10 @@ func (f *FakeAsks) pulled(data []byte) *Pulled { return nil } } + f.mu.Lock() + subject := f.subjects[string(data)] delivered := f.Delivered + f.mu.Unlock() if delivered == 0 { delivered = 1 } @@ -117,6 +133,6 @@ func (f *FakeAsks) pulled(data []byte) *Pulled { f.mu.Unlock() return nil } - return &Pulled{Data: data, Delivered: delivered, NakAfter: nakAfter, Ack: fate(&f.Acked), Nak: fate(&f.Naked), Term: fate(&f.Termed), + return &Pulled{Data: data, Subject: subject, Delivered: delivered, NakAfter: nakAfter, Ack: fate(&f.Acked), Nak: fate(&f.Naked), Term: fate(&f.Termed), InProgress: func() error { f.mu.Lock(); f.InProgress++; f.mu.Unlock(); return nil }} } diff --git a/modules/mesh-delivery-sdk/steprun/pull_integration_test.go b/modules/mesh-delivery-sdk/steprun/pull_integration_test.go index 42575013..07336764 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_integration_test.go +++ b/modules/mesh-delivery-sdk/steprun/pull_integration_test.go @@ -40,7 +40,11 @@ func publishAsk(t *testing.T, js jetstream.JetStream, a sdkstep.StepAsk) { if err != nil { t.Fatal(err) } + // A build rides main's subject; any other kind a named asker's (design 47 G §543: main takes only a build and a run). subject, _ := sdkstep.AskSubject(a.Language, a.Kind) + if a.Kind != sdkstep.AskDelivery { + subject, _ = sdkstep.AskSubjectBy(a.Language, a.Kind, "mesh-delivery-checker") + } if _, err := js.Publish(context.Background(), subject, raw); err != nil { t.Fatal(err) } @@ -197,31 +201,35 @@ func TestAnAskThatMetTheNetworkIsGivenAgainAndADroppedOneIsEnded(t *testing.T) { func contains(s, sub string) bool { return strings.Contains(s, sub) } -// Design 47 G §600: the seat's worker takes an ask put on either form of the accept subject, main's and the asker's. -func TestTheWorkerTakesBothFormsOfTheAcceptSubject(t *testing.T) { +// Design 47 G §544: main's worker takes an ask put on main's subject and the named worker one put on the asker's, the two +// filters not overlapping. +func TestTheTwoWorkersTakeTheTwoFormsOfTheAcceptSubject(t *testing.T) { js := askBus(t, 30*time.Second) stream, _ := sdkstep.AskStream(sdkstep.AskGo) worker, _ := sdkstep.AskWorker(sdkstep.AskGo, sdkstep.AskDelivery) + named, _ := sdkstep.AskNamedWorker(sdkstep.AskGo, sdkstep.AskDelivery) main, _ := sdkstep.AskSubject(sdkstep.AskGo, sdkstep.AskDelivery) - for _, subject := range []string{main, main + ".builder"} { + for _, subject := range []string{main, main + ".mesh-builds"} { if _, err := js.Publish(context.Background(), subject, []byte("{}")); err != nil { t.Fatal(err) } } - c, err := js.Consumer(context.Background(), stream, worker) - if err != nil { - t.Fatal(err) - } - batch, err := c.Fetch(2, jetstream.FetchMaxWait(3*time.Second)) - if err != nil { - t.Fatal(err) - } - var got []string - for m := range batch.Messages() { - got = append(got, m.Subject()) - _ = m.Ack() - } - if len(got) != 2 { - t.Errorf("the worker took %v, want both forms of %s", got, main) + for name, want := range map[string]string{worker: main, named: main + ".mesh-builds"} { + c, err := js.Consumer(context.Background(), stream, name) + if err != nil { + t.Fatal(err) + } + batch, err := c.Fetch(2, jetstream.FetchMaxWait(time.Second)) + if err != nil { + t.Fatal(err) + } + var got []string + for m := range batch.Messages() { + got = append(got, m.Subject()) + _ = m.Ack() + } + if len(got) != 1 || got[0] != want { + t.Errorf("%s took %v, want only %s", name, got, want) + } } } diff --git a/modules/mesh-delivery-sdk/steprun/pull_seat.go b/modules/mesh-delivery-sdk/steprun/pull_seat.go index 91e5c99b..9958e5a5 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_seat.go +++ b/modules/mesh-delivery-sdk/steprun/pull_seat.go @@ -2,6 +2,7 @@ package steprun import ( "context" + "sync" "time" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" @@ -18,10 +19,15 @@ type SeatAsks struct { Settle func(id, fate string) error // SettleAfter is stdio.SeatSettleAfter when nil (ADR 0664 §1). SettleAfter func(id string, delay time.Duration) error + // Logf says `named worker not bound` once for each queue whose named worker the membership does not list yet. + Logf func(format string, args ...any) + + mu sync.Mutex + noted map[string]bool } // Next is AskSource's. -func (s SeatAsks) Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error) { +func (s *SeatAsks) Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error) { pull, settle := s.Pull, s.Settle if pull == nil { pull = stdio.SeatPull @@ -36,7 +42,33 @@ func (s SeatAsks) Next(ctx context.Context, kind string, wait time.Duration) (*P if err := ctx.Err(); err != nil { return nil, err } - p, err := pull(worker, wait) + named, err := sdkstep.AskNamedWorker(s.Language, kind) + if err != nil { + return nil, err + } + // Main's worker first, then the named one (design 47 G §544): a holder of G pulls main's alone until its membership + // names the named worker, and says so meanwhile. A wait is shared by the two. + half := wait / 2 + p, err := pull(worker, min(wait, 0)) + if err == nil && p == nil { + var nerr error + if p, nerr = pull(named, 0); nerr != nil { + s.noteUnbound(kind, nerr) + p = nil + } else { + s.mu.Lock() + delete(s.noted, kind) + s.mu.Unlock() + } + } + if err == nil && p == nil && wait > 0 { + if p, err = pull(worker, half); err == nil && p == nil { + if p, err = pull(named, wait-half); err != nil { + s.noteUnbound(kind, err) + p, err = nil, nil + } + } + } if err != nil || p == nil { return nil, err } @@ -45,7 +77,21 @@ func (s SeatAsks) Next(ctx context.Context, kind string, wait time.Duration) (*P after = stdio.SeatSettleAfter } fate := func(f string) func() error { return func() error { return settle(p.ID, f) } } - return &Pulled{Data: p.Body, Delivered: p.Delivered, Ack: fate(stdio.SettleAck), InProgress: fate(stdio.SettleProgress), + return &Pulled{Data: p.Body, Subject: p.Subject, Delivered: p.Delivered, Ack: fate(stdio.SettleAck), InProgress: fate(stdio.SettleProgress), Nak: fate(stdio.SettleNak), Term: fate(stdio.SettleTerm), NakAfter: func(delay time.Duration) error { return after(p.ID, delay) }}, nil } + +// noteUnbound says `named worker not bound` for a queue, once until it is bound (design 47 G §544). +func (s *SeatAsks) noteUnbound(kind string, err error) { + s.mu.Lock() + if s.noted == nil { + s.noted = map[string]bool{} + } + said := s.noted[kind] + s.noted[kind] = true + s.mu.Unlock() + if !said && s.Logf != nil { + s.Logf("named worker not bound: the %s queue's named asks wait until the membership lists %s: %v", kind, "its named worker", err) + } +} diff --git a/modules/mesh-delivery-sdk/steprun/pull_seat_test.go b/modules/mesh-delivery-sdk/steprun/pull_seat_test.go index 2a4d64e6..0750b807 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_seat_test.go +++ b/modules/mesh-delivery-sdk/steprun/pull_seat_test.go @@ -18,7 +18,7 @@ import ( func TestSeatAsksPullsTheKindsWorkerAndSettlesByID(t *testing.T) { var pulled []string var settled []string - src := SeatAsks{Language: sdkstep.AskGo, + src := &SeatAsks{Language: sdkstep.AskGo, Pull: func(worker string, wait time.Duration) (*stdio.Pulled, error) { pulled = append(pulled, worker) if worker != "SEAT_NODE_BUILD_GO_CHECK_worker" { @@ -36,7 +36,7 @@ func TestSeatAsksPullsTheKindsWorkerAndSettlesByID(t *testing.T) { } _ = p.InProgress() _ = p.Ack() - if len(pulled) != 2 || pulled[0] != "SEAT_NODE_BUILD_GO_DELIVERY_worker" { + if len(pulled) != 3 || pulled[0] != "SEAT_NODE_BUILD_GO_DELIVERY_worker" || pulled[1] != "SEAT_NODE_BUILD_GO_DELIVERY_named_worker" { t.Errorf("pulled %v", pulled) } if len(settled) != 2 || settled[0] != "p9 progress" || settled[1] != "p9 ack" { @@ -146,3 +146,34 @@ func TestADroppedAskIsRefusedAtPullAndARunningOneIsEnded(t *testing.T) { } func ptr[T any](v T) *T { return &v } + +// Design 47 G §544 (BG3): a holder pulls main's worker, then the queue's named worker; while its membership does not list the +// named worker it takes main's alone and says `named worker not bound` once. +func TestSeatAsksPullsTheNamedWorkerAndSaysWhenItIsNotBound(t *testing.T) { + var said []string + bound := false + src := &SeatAsks{Language: sdkstep.AskGo, Logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }, + Pull: func(worker string, wait time.Duration) (*stdio.Pulled, error) { + if worker != "SEAT_NODE_BUILD_GO_CHECK_named_worker" { + return nil, nil + } + if !bound { + return nil, fmt.Errorf("the module pulls no work from %s", worker) + } + return &stdio.Pulled{ID: "n1", Worker: worker, Subject: "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner", Body: []byte(`{"v":1}`), Delivered: 1}, nil + }, + Settle: func(string, string) error { return nil }} + for range 2 { + if p, err := src.Next(context.Background(), sdkstep.AskCheck, 0); err != nil || p != nil { + t.Fatalf("unbound: %+v, %v", p, err) + } + } + if len(said) != 1 || !strings.Contains(said[0], "named worker not bound") { + t.Fatalf("said %q, want `named worker not bound` once", said) + } + bound = true + p, err := src.Next(context.Background(), sdkstep.AskCheck, 0) + if err != nil || p == nil || !strings.HasSuffix(p.Subject, ".mesh-pipeline-runner") { + t.Fatalf("bound: %+v, %v", p, err) + } +} From 87edff67eeb50665567c4a02bb680778d48739c7 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 09:27:01 +0100 Subject: [PATCH 3/6] B5g5-1: the indexer sends the input's chunks with buscalls.SeatChunks.Send, so its Subscribe exception goes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The acknowledgement subscription is the SDK's; the indexer's inputs.go holds none, and the test that forbids a subscription in the program no longer excepts it. Builds: design 47 G §712, §681, A-237, A-139 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-indexer/inputs.go | 20 ------- .../cmd/mesh-delivery-indexer/main.go | 2 +- .../internal/index/run_input_chunks_test.go | 10 ++-- .../internal/index/run_inputs.go | 53 ++----------------- .../mesh-delivery-indexer/manifest_test.go | 4 +- 5 files changed, 12 insertions(+), 77 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/inputs.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/inputs.go index 91ff9258..3f872366 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/inputs.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/inputs.go @@ -7,7 +7,6 @@ import ( "sync" "time" - "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -77,22 +76,3 @@ func (o *inputObjects) List() ([]stateobjects.Object, error) { } return b.List() } - -// coreChunks is the core connection as a send of chunks reaches it (index.ChunkLine): a chunk is published on no stream, an -// acknowledgement is a request the holder answers. -type coreChunks struct{ nc *nats.Conn } - -func (c coreChunks) Publish(subject string, data []byte) error { return c.nc.Publish(subject, data) } - -func (c coreChunks) Hear(subject string, fn func(string)) (func(), error) { - sub, err := c.nc.Subscribe(subject, func(m *nats.Msg) { - fn(m.Subject) - if m.Reply != "" { - _ = m.Respond([]byte("{}")) - } - }) - if err != nil { - return nil, err - } - return func() { _ = sub.Unsubscribe() }, nil -} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index cd82847e..8b0e1394 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -450,7 +450,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro inputsStore := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.InputsBucket, "no input of a run ask is kept, so the verb input answers none", logger.Printf)} inObjects := &inputObjects{js: js, module: cred.Module} stopInput, err := calls.AnswerSeat(meshcatalog.Seat, map[string]process.SeatHandler{ - meshcatalog.VerbInput: index.InputVerb(inputsStore, inObjects, coreChunks{conn}, buscalls.SeatAckWithin, logger.Printf)}) + meshcatalog.VerbInput: index.InputVerb(inputsStore, inObjects, conn, buscalls.SeatAckWithin, logger.Printf)}) if err != nil { return fmt.Errorf("answering the verb input: %w", err) } diff --git a/modules/mesh-delivery-indexer/internal/index/run_input_chunks_test.go b/modules/mesh-delivery-indexer/internal/index/run_input_chunks_test.go index bee9315a..120f2ff8 100644 --- a/modules/mesh-delivery-indexer/internal/index/run_input_chunks_test.go +++ b/modules/mesh-delivery-indexer/internal/index/run_input_chunks_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "github.com/nats-io/nats.go" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/stateobjects" @@ -42,7 +44,7 @@ func (o *objectsMem) List() ([]stateobjects.Object, error) { return nil, nil } // lineMem is a core bus in memory: a chunk published is kept, and its acknowledgement heard at once. type lineMem struct { mu sync.Mutex - hear func(string) + hear nats.MsgHandler chunks map[string][]byte } @@ -55,15 +57,15 @@ func (l *lineMem) Publish(subject string, data []byte) error { if err := json.Unmarshal(data, &c); err != nil { return err } - go h(buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbInput, Call: "0123456789abcdef"}.Ack(c.N)) + go h(&nats.Msg{Subject: buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbInput, Call: "0123456789abcdef"}.Ack(c.N)}) return nil } -func (l *lineMem) Hear(_ string, fn func(string)) (func(), error) { +func (l *lineMem) Subscribe(_ string, fn nats.MsgHandler) (*nats.Subscription, error) { l.mu.Lock() l.hear = fn l.mu.Unlock() - return func() {}, nil + return nil, nil } // Design 47 G §712, A-237: an input is of any size: over a key's limit it is kept as an object beside the state, and the verb diff --git a/modules/mesh-delivery-indexer/internal/index/run_inputs.go b/modules/mesh-delivery-indexer/internal/index/run_inputs.go index 5ecfd042..8a93c64d 100644 --- a/modules/mesh-delivery-indexer/internal/index/run_inputs.go +++ b/modules/mesh-delivery-indexer/internal/index/run_inputs.go @@ -225,20 +225,12 @@ func InputBody(ctx context.Context, st Store, objs stateobjects.Objects, digest return body, nil } -// ChunkLine is the core bus as a send of chunks reaches it: a chunk is published on its subject, kept on no stream, and each is -// acknowledged by the caller as a request on its ack subject, which the holder answers (design 47 G §681). -type ChunkLine interface { - Publish(subject string, data []byte) error - // Hear calls fn with the subject of each request made on subject (a pattern) and answers it, until stop is called. - Hear(subject string, fn func(subject string)) (stop func(), err error) -} - var callToken = regexp.MustCompile(`^[A-Za-z0-9_-]{16,64}$`) // InputVerb answers the verb input (marked chunks): the digest and how many chunks carry the bytes, then sends them in chunks of at // most buscalls.SeatChunkBytes on the call's chunk subjects, each acknowledged before the next (A-237, A-139). The input is of any // size. A caller that does not acknowledge within buscalls.SeatAckWithin ends the send, said in logf. -func InputVerb(st Store, objs stateobjects.Objects, line ChunkLine, within time.Duration, logf func(string, ...any)) func(caller string, body json.RawMessage) (any, error) { +func InputVerb(st Store, objs stateobjects.Objects, line buscalls.ChunkLine, within time.Duration, logf func(string, ...any)) func(caller string, body json.RawMessage) (any, error) { return func(_ string, raw json.RawMessage) (any, error) { var in meshcatalog.InputInput if err := json.Unmarshal(raw, &in); err != nil { @@ -253,31 +245,10 @@ func InputVerb(st Store, objs stateobjects.Objects, line ChunkLine, within time. } send := buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbInput, Call: in.Call} total := int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes) - acks := make(chan string, 1) - // heard before the answer goes, since the caller acknowledges only after it - stop, err := line.Hear(buscalls.SeatAcks(meshcatalog.Seat, meshcatalog.VerbInput), func(subject string) { - select { - case acks <- subject: - default: - } - }) - if err != nil { - return nil, fmt.Errorf("the acknowledgements cannot be heard, so no chunk is sent: %w", err) - } + // the send subscribes to the acknowledgements itself and ends when it has sent every chunk or one is not acknowledged go func() { - defer stop() - for n := int64(1); n <= total; n++ { - from := (n - 1) * buscalls.SeatChunkBytes - to := min(int64(len(body)), from+buscalls.SeatChunkBytes) - data, _ := json.Marshal(buscalls.SeatChunk{N: n, Of: total, Data: body[from:to]}) - if err := line.Publish(send.Chunk(n), data); err != nil { - logf("the send %s of the input %s ends at chunk %d of %d: %v", in.Call, in.Digest, n, total, err) - return - } - if !acknowledged(acks, send.Ack(n), within) { - logf("the send %s of the input %s ends at chunk %d of %d: no acknowledgement within %s", in.Call, in.Digest, n, total, within) - return - } + if err := send.Send(line, body, within, logf); err != nil { + logf("the send %s of the input %s: %v", in.Call, in.Digest, err) } }() return meshcatalog.InputAnswer{Digest: in.Digest, Chunks: total}, nil @@ -339,19 +310,3 @@ func InputsReadTool(st Store, objs stateobjects.Objects) stdio.Tool { return out, nil }} } - -// acknowledged waits within for the acknowledgement on subject, skipping another call's. -func acknowledged(acks <-chan string, subject string, within time.Duration) bool { - ctx, cancel := context.WithTimeout(context.Background(), within) - defer cancel() - for { - select { - case got := <-acks: - if got == subject { - return true - } - case <-ctx.Done(): - return false - } - } -} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index d7da242e..2eba8cea 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -382,9 +382,7 @@ func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1238/AC9 (ADR for _, f := range forbidden { // The seat list reader's Bus interface names a method Subscribe; the indexer's own bus hands it its own events // directly and subscribes to nothing on the bus. - // The acknowledgements of the chunks of the verb input are heard on the core connection while a send runs (design 47 G - // §712, A-237, §681): cmd/mesh-delivery-indexer/inputs.go. - if f == "Subscribe(" && (strings.HasSuffix(path, "seatlist.go") || strings.HasSuffix(path, "mesh-delivery-indexer/inputs.go")) { + if f == "Subscribe(" && strings.HasSuffix(path, "seatlist.go") { continue } if strings.Contains(string(src), f) { From f06c9cf5fdfebd3fe4a9f210e19cf2e588227032 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 09:34:48 +0100 Subject: [PATCH 4/6] C1c-1: the three build seats' verb output answers a run's answer in chunks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The part build-step gains the verb `output` (the ask and the call in, the ask and the chunk count out, marked chunks, read by the askers of a run), and the fields output and output-object of step-ended name the run's answer inline or as an object the holder keeps. The three seats' fixtures carry the verb; the generated Go and TypeScript follow. Builds: design 47 G A-254, A-139, §712, §354 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../go/seats/nodebuildgo/contract.go | 23 ++++++-- .../go/seats/nodebuildgo/contract_test.go | 4 ++ .../go/seats/nodebuildimage/contract.go | 23 ++++++-- .../go/seats/nodebuildimage/contract_test.go | 4 ++ .../go/seats/nodebuildnode/contract.go | 23 ++++++-- .../go/seats/nodebuildnode/contract_test.go | 4 ++ .../parts/build-step.json | 53 ++++++++++++++++++- .../seats/node-build-go/fixtures/verbs.json | 16 ++++++ .../node-build-image/fixtures/verbs.json | 16 ++++++ .../seats/node-build-node/fixtures/verbs.json | 16 ++++++ .../ts/seats/nodebuildgo/index.d.ts | 3 ++ .../ts/seats/nodebuildgo/index.js | 1 + .../seats/nodebuildgo/parts/build-step.d.ts | 21 +++++++- .../ts/seats/nodebuildgo/parts/build-step.js | 1 + .../ts/seats/nodebuildimage/index.d.ts | 3 ++ .../ts/seats/nodebuildimage/index.js | 1 + .../nodebuildimage/parts/build-step.d.ts | 21 +++++++- .../seats/nodebuildimage/parts/build-step.js | 1 + .../ts/seats/nodebuildnode/index.d.ts | 3 ++ .../ts/seats/nodebuildnode/index.js | 1 + .../seats/nodebuildnode/parts/build-step.d.ts | 21 +++++++- .../seats/nodebuildnode/parts/build-step.js | 1 + 22 files changed, 243 insertions(+), 17 deletions(-) diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go index 7db4425a..19547865 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go @@ -21,6 +21,7 @@ const ( VerbDrop = "drop" VerbKeptArtifact = "kept-artifact" VerbSteps = "steps" + VerbOutput = "output" ) const ( @@ -41,7 +42,7 @@ const ( // Verbs is every verb the seat serves, in its definition's order: a new copy each call. func Verbs() []string { - return []string{VerbCurrent, VerbPause, VerbResume, VerbDrop, VerbKeptArtifact, VerbSteps} + return []string{VerbCurrent, VerbPause, VerbResume, VerbDrop, VerbKeptArtifact, VerbSteps, VerbOutput} } // Events is every event published on the seat's subject: a new copy each call. @@ -804,9 +805,9 @@ type StepEnded struct { Node string `json:"node"` // never passed for a step that ran nothing Outcome string `json:"outcome"` - // the step's output when it fits the message + // the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254) Output string `json:"output,omitempty"` - // the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16) + // the name of the object that holds the run's answer, kept by the build seat's holder, when the answer is over what the event carries (design 47 G A-254); read through the verb output OutputObject string `json:"output-object,omitempty"` Pipeline string `json:"pipeline,omitempty"` Rerun bool `json:"rerun,omitempty"` @@ -1064,6 +1065,22 @@ type StepsAnswer struct { Steps []DetectedStep `json:"steps,omitempty"` } +// OutputInput is what output takes. +type OutputInput struct { + // the ask's id, the `output-object` of its step-ended naming it + Ask string `json:"ask"` + // 128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements + Call string `json:"call"` +} + +// OutputAnswer is what output answers. +type OutputAnswer struct { + // the ask's id + Ask string `json:"ask"` + // how many chunks are sent; 0 when this node keeps no answer of that ask + Chunks int64 `json:"chunks"` +} + // DeliveryAccept is the body of delivery. type DeliveryAccept = StepAsk diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract_test.go b/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract_test.go index 50eb29a1..13ef5448 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract_test.go @@ -121,6 +121,10 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "steps answers a drop that the hand-act log refused", subject: "", given: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", wire: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "output asks a run's answer by its ask", subject: "", given: "{\"ask\":\"m2kx9a1b3c4d5f\",\"call\":\"0123456789abcdef0123456789abcdef\"}", wire: "{\"ask\":\"m2kx9a1b3c4d5f\",\"call\":\"0123456789abcdef0123456789abcdef\"}", + decode: func(b []byte) (any, error) { var v OutputInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "output answers how many chunks carry the answer", subject: "", given: "{\"ask\":\"m2kx9a1b3c4d5f\",\"chunks\":3}", wire: "{\"ask\":\"m2kx9a1b3c4d5f\",\"chunks\":3}", + decode: func(b []byte) (any, error) { var v OutputAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, } for _, c := range cases { v, err := c.decode([]byte(c.given)) diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go index 367353bb..c0a708f9 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go @@ -21,6 +21,7 @@ const ( VerbDrop = "drop" VerbKeptArtifact = "kept-artifact" VerbSteps = "steps" + VerbOutput = "output" ) const ( @@ -40,7 +41,7 @@ const ( // Verbs is every verb the seat serves, in its definition's order: a new copy each call. func Verbs() []string { - return []string{VerbCurrent, VerbPause, VerbResume, VerbDrop, VerbKeptArtifact, VerbSteps} + return []string{VerbCurrent, VerbPause, VerbResume, VerbDrop, VerbKeptArtifact, VerbSteps, VerbOutput} } // Events is every event published on the seat's subject: a new copy each call. @@ -802,9 +803,9 @@ type StepEnded struct { Node string `json:"node"` // never passed for a step that ran nothing Outcome string `json:"outcome"` - // the step's output when it fits the message + // the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254) Output string `json:"output,omitempty"` - // the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16) + // the name of the object that holds the run's answer, kept by the build seat's holder, when the answer is over what the event carries (design 47 G A-254); read through the verb output OutputObject string `json:"output-object,omitempty"` Pipeline string `json:"pipeline,omitempty"` Rerun bool `json:"rerun,omitempty"` @@ -1062,6 +1063,22 @@ type StepsAnswer struct { Steps []DetectedStep `json:"steps,omitempty"` } +// OutputInput is what output takes. +type OutputInput struct { + // the ask's id, the `output-object` of its step-ended naming it + Ask string `json:"ask"` + // 128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements + Call string `json:"call"` +} + +// OutputAnswer is what output answers. +type OutputAnswer struct { + // the ask's id + Ask string `json:"ask"` + // how many chunks are sent; 0 when this node keeps no answer of that ask + Chunks int64 `json:"chunks"` +} + // DeliveryAccept is the body of delivery. type DeliveryAccept = StepAsk diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract_test.go b/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract_test.go index 5ebe76a8..acbff00c 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract_test.go @@ -103,6 +103,10 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "steps answers a drop that the hand-act log refused", subject: "", given: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", wire: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "output asks a run's answer by its ask", subject: "", given: "{\"ask\":\"m2kx9a1b3c4d5f\",\"call\":\"0123456789abcdef0123456789abcdef\"}", wire: "{\"ask\":\"m2kx9a1b3c4d5f\",\"call\":\"0123456789abcdef0123456789abcdef\"}", + decode: func(b []byte) (any, error) { var v OutputInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "output answers how many chunks carry the answer", subject: "", given: "{\"ask\":\"m2kx9a1b3c4d5f\",\"chunks\":3}", wire: "{\"ask\":\"m2kx9a1b3c4d5f\",\"chunks\":3}", + decode: func(b []byte) (any, error) { var v OutputAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, } for _, c := range cases { v, err := c.decode([]byte(c.given)) diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go index 6014fe4a..ddc32629 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go @@ -21,6 +21,7 @@ const ( VerbDrop = "drop" VerbKeptArtifact = "kept-artifact" VerbSteps = "steps" + VerbOutput = "output" ) const ( @@ -40,7 +41,7 @@ const ( // Verbs is every verb the seat serves, in its definition's order: a new copy each call. func Verbs() []string { - return []string{VerbCurrent, VerbPause, VerbResume, VerbDrop, VerbKeptArtifact, VerbSteps} + return []string{VerbCurrent, VerbPause, VerbResume, VerbDrop, VerbKeptArtifact, VerbSteps, VerbOutput} } // Events is every event published on the seat's subject: a new copy each call. @@ -802,9 +803,9 @@ type StepEnded struct { Node string `json:"node"` // never passed for a step that ran nothing Outcome string `json:"outcome"` - // the step's output when it fits the message + // the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254) Output string `json:"output,omitempty"` - // the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16) + // the name of the object that holds the run's answer, kept by the build seat's holder, when the answer is over what the event carries (design 47 G A-254); read through the verb output OutputObject string `json:"output-object,omitempty"` Pipeline string `json:"pipeline,omitempty"` Rerun bool `json:"rerun,omitempty"` @@ -1062,6 +1063,22 @@ type StepsAnswer struct { Steps []DetectedStep `json:"steps,omitempty"` } +// OutputInput is what output takes. +type OutputInput struct { + // the ask's id, the `output-object` of its step-ended naming it + Ask string `json:"ask"` + // 128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements + Call string `json:"call"` +} + +// OutputAnswer is what output answers. +type OutputAnswer struct { + // the ask's id + Ask string `json:"ask"` + // how many chunks are sent; 0 when this node keeps no answer of that ask + Chunks int64 `json:"chunks"` +} + // DeliveryAccept is the body of delivery. type DeliveryAccept = StepAsk diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract_test.go b/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract_test.go index 78aff7ee..55f5c91e 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract_test.go @@ -107,6 +107,10 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "steps answers a drop that the hand-act log refused", subject: "", given: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", wire: "{\"error\":\"the hand-act log refused: nothing was deleted\"}", decode: func(b []byte) (any, error) { var v StepsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "output asks a run's answer by its ask", subject: "", given: "{\"ask\":\"m2kx9a1b3c4d5f\",\"call\":\"0123456789abcdef0123456789abcdef\"}", wire: "{\"ask\":\"m2kx9a1b3c4d5f\",\"call\":\"0123456789abcdef0123456789abcdef\"}", + decode: func(b []byte) (any, error) { var v OutputInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, + {name: "output answers how many chunks carry the answer", subject: "", given: "{\"ask\":\"m2kx9a1b3c4d5f\",\"chunks\":3}", wire: "{\"ask\":\"m2kx9a1b3c4d5f\",\"chunks\":3}", + decode: func(b []byte) (any, error) { var v OutputAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, } for _, c := range cases { v, err := c.decode([]byte(c.given)) diff --git a/modules/mesh-delivery-contracts/parts/build-step.json b/modules/mesh-delivery-contracts/parts/build-step.json index 8c0b00b6..8f1bd487 100644 --- a/modules/mesh-delivery-contracts/parts/build-step.json +++ b/modules/mesh-delivery-contracts/parts/build-step.json @@ -233,6 +233,55 @@ "required": [] }, "class": "act" + }, + { + "name": "output", + "class": "read", + "callers": [ + "seat:mesh-catalog", + "seat:mesh-wants", + "seat:mesh-delivery-checker", + "seat:mesh-pipeline-runner" + ], + "description": "A run's answer by its ask, when the step-ended of the run names it as `output-object` because the answer is over what the event carries (design 47 G A-254): the answer's bytes in chunks of at most 512 KiB on the chunk subject of the call, each acknowledged by the caller on the ack subject within 30 seconds; the answer is of any size. An answer inline in the event is never asked here.", + "input": { + "type": "object", + "properties": { + "ask": { + "type": "string", + "description": "the ask's id, the `output-object` of its step-ended naming it", + "minLength": 1 + }, + "call": { + "type": "string", + "description": "128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements", + "minLength": 1 + } + }, + "required": [ + "ask", + "call" + ] + }, + "answer": { + "type": "object", + "properties": { + "ask": { + "type": "string", + "description": "the ask's id" + }, + "chunks": { + "type": "integer", + "minimum": 0, + "description": "how many chunks are sent; 0 when this node keeps no answer of that ask" + } + }, + "required": [ + "ask", + "chunks" + ] + }, + "chunks": true } ], "defs": { @@ -1595,11 +1644,11 @@ }, "output": { "type": "string", - "description": "the step's output when it fits the message" + "description": "the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254)" }, "output-object": { "type": "string", - "description": "the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16)" + "description": "the name of the object that holds the run's answer, kept by the build seat's holder, when the answer is over what the event carries (design 47 G A-254); read through the verb output" }, "rerun": { "type": "boolean" diff --git a/modules/mesh-delivery-contracts/seats/node-build-go/fixtures/verbs.json b/modules/mesh-delivery-contracts/seats/node-build-go/fixtures/verbs.json index 7a02ff62..070ce00d 100644 --- a/modules/mesh-delivery-contracts/seats/node-build-go/fixtures/verbs.json +++ b/modules/mesh-delivery-contracts/seats/node-build-go/fixtures/verbs.json @@ -124,5 +124,21 @@ "given": { "error": "the hand-act log refused: nothing was deleted" } + }, + { + "name": "output asks a run's answer by its ask", + "of": "verb:output.input", + "given": { + "ask": "m2kx9a1b3c4d5f", + "call": "0123456789abcdef0123456789abcdef" + } + }, + { + "name": "output answers how many chunks carry the answer", + "of": "verb:output.answer", + "given": { + "ask": "m2kx9a1b3c4d5f", + "chunks": 3 + } } ] diff --git a/modules/mesh-delivery-contracts/seats/node-build-image/fixtures/verbs.json b/modules/mesh-delivery-contracts/seats/node-build-image/fixtures/verbs.json index 7a02ff62..070ce00d 100644 --- a/modules/mesh-delivery-contracts/seats/node-build-image/fixtures/verbs.json +++ b/modules/mesh-delivery-contracts/seats/node-build-image/fixtures/verbs.json @@ -124,5 +124,21 @@ "given": { "error": "the hand-act log refused: nothing was deleted" } + }, + { + "name": "output asks a run's answer by its ask", + "of": "verb:output.input", + "given": { + "ask": "m2kx9a1b3c4d5f", + "call": "0123456789abcdef0123456789abcdef" + } + }, + { + "name": "output answers how many chunks carry the answer", + "of": "verb:output.answer", + "given": { + "ask": "m2kx9a1b3c4d5f", + "chunks": 3 + } } ] diff --git a/modules/mesh-delivery-contracts/seats/node-build-node/fixtures/verbs.json b/modules/mesh-delivery-contracts/seats/node-build-node/fixtures/verbs.json index 7a02ff62..070ce00d 100644 --- a/modules/mesh-delivery-contracts/seats/node-build-node/fixtures/verbs.json +++ b/modules/mesh-delivery-contracts/seats/node-build-node/fixtures/verbs.json @@ -124,5 +124,21 @@ "given": { "error": "the hand-act log refused: nothing was deleted" } + }, + { + "name": "output asks a run's answer by its ask", + "of": "verb:output.input", + "given": { + "ask": "m2kx9a1b3c4d5f", + "call": "0123456789abcdef0123456789abcdef" + } + }, + { + "name": "output answers how many chunks carry the answer", + "of": "verb:output.answer", + "given": { + "ask": "m2kx9a1b3c4d5f", + "chunks": 3 + } } ] diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/index.d.ts index 1fe4b734..de2a047f 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/index.d.ts @@ -13,6 +13,7 @@ export declare const verbs: { readonly drop: "drop"; readonly keptArtifact: "kept-artifact"; readonly steps: "steps"; + readonly output: "output"; }; export declare const accepts: { @@ -146,6 +147,8 @@ export type LevelAnswerDigestPackagesItem = buildStep.LevelAnswerDigestPackagesI export type LevelAnswerKeptItem = buildStep.LevelAnswerKeptItem; export type LevelAnswerUnmetItem = buildStep.LevelAnswerUnmetItem; export type ModuleTree = buildStep.ModuleTree; +export type OutputAnswer = buildStep.OutputAnswer; +export type OutputInput = buildStep.OutputInput; export type Pause = buildStep.Pause; export type PauseAnswer = buildStep.PauseAnswer; export type PauseInput = buildStep.PauseInput; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/index.js b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/index.js index f2566f56..a007e1cb 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/index.js @@ -11,6 +11,7 @@ export const verbs = { drop: "drop", keptArtifact: "kept-artifact", steps: "steps", + output: "output", }; export const accepts = { diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts index 2addf5c4..e6968656 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts @@ -9,6 +9,7 @@ export declare const verbs: { readonly drop: "drop"; readonly keptArtifact: "kept-artifact"; readonly steps: "steps"; + readonly output: "output"; }; export declare const events: { @@ -432,9 +433,9 @@ export interface StepEnded { "node": string; /** never passed for a step that ran nothing */ "outcome": "passed" | "failed" | "nothing to run" | "could not run"; - /** the step's output when it fits the message */ + /** the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254) */ "output"?: string; - /** the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16) */ + /** the name of the object that holds the run's answer, kept by the build seat's holder, when the answer is over what the event carries (design 47 G A-254); read through the verb output */ "output-object"?: string; "pipeline"?: string; "rerun"?: boolean; @@ -655,6 +656,22 @@ export interface StepsAnswer { "steps"?: DetectedStep[]; } +/** is what output takes. */ +export interface OutputInput { + /** the ask's id, the `output-object` of its step-ended naming it */ + "ask": string; + /** 128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements */ + "call": string; +} + +/** is what output answers. */ +export interface OutputAnswer { + /** the ask's id */ + "ask": string; + /** how many chunks are sent; 0 when this node keeps no answer of that ask */ + "chunks": number; +} + /** is the body of the event step-ended. */ export type StepEndedEvent = StepEnded; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.js b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.js index 400cdf1c..2d91b9b6 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.js +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.js @@ -9,6 +9,7 @@ export const verbs = { drop: "drop", keptArtifact: "kept-artifact", steps: "steps", + output: "output", }; export const events = { diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/index.d.ts index dff95615..c6790130 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/index.d.ts @@ -13,6 +13,7 @@ export declare const verbs: { readonly drop: "drop"; readonly keptArtifact: "kept-artifact"; readonly steps: "steps"; + readonly output: "output"; }; export declare const accepts: { @@ -143,6 +144,8 @@ export type LevelAnswerDigestPackagesItem = buildStep.LevelAnswerDigestPackagesI export type LevelAnswerKeptItem = buildStep.LevelAnswerKeptItem; export type LevelAnswerUnmetItem = buildStep.LevelAnswerUnmetItem; export type ModuleTree = buildStep.ModuleTree; +export type OutputAnswer = buildStep.OutputAnswer; +export type OutputInput = buildStep.OutputInput; export type Pause = buildStep.Pause; export type PauseAnswer = buildStep.PauseAnswer; export type PauseInput = buildStep.PauseInput; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/index.js b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/index.js index bf0ad43c..17048b39 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/index.js @@ -11,6 +11,7 @@ export const verbs = { drop: "drop", keptArtifact: "kept-artifact", steps: "steps", + output: "output", }; export const accepts = { diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts index 2addf5c4..e6968656 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts @@ -9,6 +9,7 @@ export declare const verbs: { readonly drop: "drop"; readonly keptArtifact: "kept-artifact"; readonly steps: "steps"; + readonly output: "output"; }; export declare const events: { @@ -432,9 +433,9 @@ export interface StepEnded { "node": string; /** never passed for a step that ran nothing */ "outcome": "passed" | "failed" | "nothing to run" | "could not run"; - /** the step's output when it fits the message */ + /** the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254) */ "output"?: string; - /** the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16) */ + /** the name of the object that holds the run's answer, kept by the build seat's holder, when the answer is over what the event carries (design 47 G A-254); read through the verb output */ "output-object"?: string; "pipeline"?: string; "rerun"?: boolean; @@ -655,6 +656,22 @@ export interface StepsAnswer { "steps"?: DetectedStep[]; } +/** is what output takes. */ +export interface OutputInput { + /** the ask's id, the `output-object` of its step-ended naming it */ + "ask": string; + /** 128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements */ + "call": string; +} + +/** is what output answers. */ +export interface OutputAnswer { + /** the ask's id */ + "ask": string; + /** how many chunks are sent; 0 when this node keeps no answer of that ask */ + "chunks": number; +} + /** is the body of the event step-ended. */ export type StepEndedEvent = StepEnded; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.js b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.js index 400cdf1c..2d91b9b6 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.js +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.js @@ -9,6 +9,7 @@ export const verbs = { drop: "drop", keptArtifact: "kept-artifact", steps: "steps", + output: "output", }; export const events = { diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/index.d.ts index 7d0eaaf3..59f62b62 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/index.d.ts @@ -13,6 +13,7 @@ export declare const verbs: { readonly drop: "drop"; readonly keptArtifact: "kept-artifact"; readonly steps: "steps"; + readonly output: "output"; }; export declare const accepts: { @@ -143,6 +144,8 @@ export type LevelAnswerDigestPackagesItem = buildStep.LevelAnswerDigestPackagesI export type LevelAnswerKeptItem = buildStep.LevelAnswerKeptItem; export type LevelAnswerUnmetItem = buildStep.LevelAnswerUnmetItem; export type ModuleTree = buildStep.ModuleTree; +export type OutputAnswer = buildStep.OutputAnswer; +export type OutputInput = buildStep.OutputInput; export type Pause = buildStep.Pause; export type PauseAnswer = buildStep.PauseAnswer; export type PauseInput = buildStep.PauseInput; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/index.js b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/index.js index 90d22dca..f5591f99 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/index.js @@ -11,6 +11,7 @@ export const verbs = { drop: "drop", keptArtifact: "kept-artifact", steps: "steps", + output: "output", }; export const accepts = { diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts index 2addf5c4..e6968656 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts @@ -9,6 +9,7 @@ export declare const verbs: { readonly drop: "drop"; readonly keptArtifact: "kept-artifact"; readonly steps: "steps"; + readonly output: "output"; }; export declare const events: { @@ -432,9 +433,9 @@ export interface StepEnded { "node": string; /** never passed for a step that ran nothing */ "outcome": "passed" | "failed" | "nothing to run" | "could not run"; - /** the step's output when it fits the message */ + /** the run's answer (its program's standard output's last line) when it fits the event (design 47 G A-254) */ "output"?: string; - /** the output's object in the sender's sent store, when over the per-message limit (ADR 0666 section 16) */ + /** the name of the object that holds the run's answer, kept by the build seat's holder, when the answer is over what the event carries (design 47 G A-254); read through the verb output */ "output-object"?: string; "pipeline"?: string; "rerun"?: boolean; @@ -655,6 +656,22 @@ export interface StepsAnswer { "steps"?: DetectedStep[]; } +/** is what output takes. */ +export interface OutputInput { + /** the ask's id, the `output-object` of its step-ended naming it */ + "ask": string; + /** 128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements */ + "call": string; +} + +/** is what output answers. */ +export interface OutputAnswer { + /** the ask's id */ + "ask": string; + /** how many chunks are sent; 0 when this node keeps no answer of that ask */ + "chunks": number; +} + /** is the body of the event step-ended. */ export type StepEndedEvent = StepEnded; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.js b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.js index 400cdf1c..2d91b9b6 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.js +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.js @@ -9,6 +9,7 @@ export const verbs = { drop: "drop", keptArtifact: "kept-artifact", steps: "steps", + output: "output", }; export const events = { From 7125a9f92f1f3ce3e0ebcd4b4eff00fa83fe085a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 09:37:51 +0100 Subject: [PATCH 5/6] C1c-2a: the step runner says a run's answer in step-ended, inline or as an object, and serves it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The puller's end of a run carries the run's program's last line of standard output as `output` up to StepEndedInlineMost, else keeps it with KeepOutput and names the object in `output-object`; a program that exits other than 0 ends failed. OutputTool serves the seat's verb `output` from the kept object, as the whole file the tool runner sends in chunks; OutputsStore names the store. Builds: design 47 G A-254, A-257, §602 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../buildstep/pipelinestep.go | 11 +++ modules/mesh-delivery-sdk/steprun/askers.go | 2 +- modules/mesh-delivery-sdk/steprun/outputs.go | 42 ++++++++++ .../mesh-delivery-sdk/steprun/outputs_test.go | 42 ++++++++++ modules/mesh-delivery-sdk/steprun/pull.go | 49 +++++++++-- .../steprun/run_output_test.go | 83 +++++++++++++++++++ .../mesh-delivery-sdk/steprun/seat_tools.go | 30 +++++++ .../steprun/seat_tools_test.go | 24 ++++++ 8 files changed, 276 insertions(+), 7 deletions(-) create mode 100644 modules/mesh-delivery-sdk/steprun/outputs.go create mode 100644 modules/mesh-delivery-sdk/steprun/outputs_test.go create mode 100644 modules/mesh-delivery-sdk/steprun/run_output_test.go diff --git a/modules/mesh-delivery-sdk/buildstep/pipelinestep.go b/modules/mesh-delivery-sdk/buildstep/pipelinestep.go index 40e51f74..cee86f4a 100644 --- a/modules/mesh-delivery-sdk/buildstep/pipelinestep.go +++ b/modules/mesh-delivery-sdk/buildstep/pipelinestep.go @@ -39,6 +39,14 @@ var StepOutcomes = []string{OutcomePassed, OutcomeFailed, OutcomeNothingToRun, O // StepEndedVersion is the layout version of a step's end. const StepEndedVersion = 1 +// StepEndedInlineMost is the most bytes of a run's answer a step-ended carries inline as `output`: the bus's message limit +// less the room its headers take and 16 KiB for the end's other fields. An answer over it is kept by the holder as an object, +// which `output-object` names and the seat's verb `output` answers (design 47 G A-254). +const StepEndedInlineMost = (1 << 20) - (8 << 10) - (16 << 10) + +// OutputObjectName is the name of the object a build seat's holder keeps a run's answer in, by the ask's id. +func OutputObjectName(ask string) string { return "output." + ask } + // KeptResult is a package's passing result at the digest of what its tests read (ADR 0666 §8). type KeptResult struct { Package string `json:"package"` @@ -154,6 +162,9 @@ func (e StepEnded) check() error { if err := checkNode(e.Node); err != nil { return err } + if e.Output != "" && e.OutputObject != "" { + return fmt.Errorf("buildstep: the end of %s carries its answer inline and as an object", e.ID) + } if e.Pipeline == "" { // G §548, §602: the end of an ask outside a pipeline names its asker, the ask and when, and no commit or // fingerprint. diff --git a/modules/mesh-delivery-sdk/steprun/askers.go b/modules/mesh-delivery-sdk/steprun/askers.go index 3023ad38..eb2c77c6 100644 --- a/modules/mesh-delivery-sdk/steprun/askers.go +++ b/modules/mesh-delivery-sdk/steprun/askers.go @@ -106,7 +106,7 @@ func (p *Puller) refuseAsker(msg *Pulled, ask sdkstep.StepAsk, why string) { } p.logf("the ask %s is refused on %s: %s", ask.ID, p.Node, why) if ask.Step == nil { - p.sayEnded(msg, ask, sdkstep.OutcomeFailed, why) + p.sayEnded(msg, ask, sdkstep.OutcomeFailed, why, nil) } else if end, err := sdkstep.EndOf(ask, p.Node, sdkstep.OutcomeFailed); err == nil { // A pipeline step's end names its pipeline, step, commit and fingerprint (its own form, §602). end.Said, end.At, end.Asker = why, now().UTC(), askerOf(msg, ask) diff --git a/modules/mesh-delivery-sdk/steprun/outputs.go b/modules/mesh-delivery-sdk/steprun/outputs.go new file mode 100644 index 00000000..b228cf89 --- /dev/null +++ b/modules/mesh-delivery-sdk/steprun/outputs.go @@ -0,0 +1,42 @@ +package steprun + +import ( + "fmt" + + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// OutputsStore is the local name of the object store a build seat's holder keeps a run's answer in when it is over what +// step-ended carries inline (design 47 G A-254), as the holder's manifest `requires` asks for it with the template objects. +const OutputsStore = "outputs" + +// OutputObjects is the object store of OutputsStore as a holder uses it (mesh-sdk busstate.ObjectsOf(OutputsStore)). +type OutputObjects interface { + Put(name string, data []byte, description string) (buscalls.ObjectInfo, error) + Get(name string) (buscalls.ObjectData, error) +} + +// KeepOutputIn is a holder's Puller.KeepOutput over its store of outputs. +func KeepOutputIn(o OutputObjects) func(object string, body []byte) error { + return func(object string, body []byte) error { + if o == nil { + return fmt.Errorf("this module keeps no %s store", OutputsStore) + } + _, err := o.Put(object, body, "a run's answer, named by its step-ended") + return err + } +} + +// ReadOutputFrom is a holder's read of a kept answer for OutputTool over its store of outputs. +func ReadOutputFrom(o OutputObjects) func(object string) ([]byte, error) { + return func(object string) ([]byte, error) { + if o == nil { + return nil, fmt.Errorf("this module keeps no %s store", OutputsStore) + } + got, err := o.Get(object) + if err != nil { + return nil, fmt.Errorf("the answer %s in the %s store: %w", object, OutputsStore, err) + } + return got.Data, nil + } +} diff --git a/modules/mesh-delivery-sdk/steprun/outputs_test.go b/modules/mesh-delivery-sdk/steprun/outputs_test.go new file mode 100644 index 00000000..5013b04d --- /dev/null +++ b/modules/mesh-delivery-sdk/steprun/outputs_test.go @@ -0,0 +1,42 @@ +package steprun + +import ( + "errors" + "testing" + + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +type fakeOutputs struct{ m map[string][]byte } + +func (f fakeOutputs) Put(name string, data []byte, _ string) (buscalls.ObjectInfo, error) { + f.m[name] = data + return buscalls.ObjectInfo{}, nil +} +func (f fakeOutputs) Get(name string) (buscalls.ObjectData, error) { + d, ok := f.m[name] + if !ok { + return buscalls.ObjectData{}, errors.New("object not found") + } + return buscalls.ObjectData{Data: d}, nil +} + +// Design 47 G A-254: what a holder keeps through KeepOutputIn it reads again through ReadOutputFrom; with no store both refuse by name. +func TestOutputsKeptAreRead(t *testing.T) { + st := fakeOutputs{m: map[string][]byte{}} + if err := KeepOutputIn(st)("output.a1", []byte("x")); err != nil { + t.Fatal(err) + } + if got, err := ReadOutputFrom(st)("output.a1"); err != nil || string(got) != "x" { + t.Fatalf("read %q, %v", got, err) + } + if _, err := ReadOutputFrom(st)("output.none"); err == nil { + t.Fatal("an absent answer was read") + } + if KeepOutputIn(nil)("o", nil) == nil { + t.Fatal("no store kept an answer") + } + if _, err := ReadOutputFrom(nil)("o"); err == nil { + t.Fatal("no store read an answer") + } +} diff --git a/modules/mesh-delivery-sdk/steprun/pull.go b/modules/mesh-delivery-sdk/steprun/pull.go index 77a772d6..f95a8718 100644 --- a/modules/mesh-delivery-sdk/steprun/pull.go +++ b/modules/mesh-delivery-sdk/steprun/pull.go @@ -81,6 +81,10 @@ type Puller struct { // every ProgressEvery while it runs, step-ended when an ask outside a pipeline ends (design 47 G §496, §602). Nil says // none, a test's. ProgressEvery defaults to DefaultProgressEvery. Say func(event string, body any, id string) error + // KeepOutput keeps a run's answer over what step-ended carries inline (sdkstep.StepEndedInlineMost) as the object named + // object, which the seat's verb `output` answers (design 47 G A-254); a holder keeps it in the object store its manifest + // requires. Nil keeps none: such an answer ends the ask failed, saying so. + KeepOutput func(object string, body []byte) error // Callers says which seats the build seat's definition lists as callers of an accepted kind (CallersFromDefinitions); // found is false while the seat list holds no such kind. Nil checks no named asker (design 47 G §543, A-163). Callers func(kind string) (callers []string, found bool) @@ -753,7 +757,7 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key var dec *Declining if errors.As(err, &dec) && ctx.Err() == nil { p.declineRun(msg, ask, key, step, dec) - p.sayEnded(msg, ask, sdkstep.OutcomeCouldNotRun, dec.Text) + p.sayEnded(msg, ask, sdkstep.OutcomeCouldNotRun, dec.Text, nil) return } if ctx.Err() != nil { @@ -791,9 +795,9 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key if len(outcome) > 300 { outcome = outcome[:300] + "..." } - p.sayEnded(msg, ask, sdkstep.OutcomeFailed, err.Error()) + p.sayEnded(msg, ask, sdkstep.OutcomeFailed, err.Error(), nil) } else { - p.sayEnded(msg, ask, sdkstep.OutcomePassed, "") + p.sayEnded(msg, ask, sdkstep.OutcomePassed, "", result) } p.logf("the %s ask %s ended on %s after %s, in its phase %q; %s", ask.Kind, ask.ID, p.Node, now().Sub(step.Started).Round(time.Second), step.Phase, outcome) @@ -976,18 +980,35 @@ func (p *Puller) say(event string, body any, id string) { } // sayEnded says step-ended for an ask outside a pipeline, whatever its asker (design 47 G §602): a pipeline step's end is -// its run's own (AskSteps.SayStep). -func (p *Puller) sayEnded(msg *Pulled, ask sdkstep.StepAsk, outcome, said string) { +// its run's own (AskSteps.SayStep). The end of a run carries the run's answer (A-254): the last line of its program's +// standard output, inline as `output` up to sdkstep.StepEndedInlineMost, else as `output-object`, kept by KeepOutput; a +// program that exits other than 0 ends failed with its answer carried. +func (p *Puller) sayEnded(msg *Pulled, ask sdkstep.StepAsk, outcome, said string, result any) { if p.Say == nil || ask.Step != nil { return } + end := sdkstep.StepEnded{ID: ask.ID, Asker: askerOf(msg, ask), Ask: ask.ID, Node: p.Node, Outcome: outcome, At: now().UTC()} + if ans, ok := runAnswerOf(ask, result); ok { + if ans.Exit != 0 && outcome == sdkstep.OutcomePassed { + outcome, said = sdkstep.OutcomeFailed, fmt.Sprintf("the program exited %d: %s", ans.Exit, ans.Said) + } + if len(ans.Output) <= sdkstep.StepEndedInlineMost { + end.Output = string(ans.Output) + } else if object := sdkstep.OutputObjectName(ask.ID); p.KeepOutput == nil { + outcome, said = sdkstep.OutcomeFailed, fmt.Sprintf("the run's answer is %d bytes, over what the end carries, and this holder keeps no object for it", len(ans.Output)) + } else if err := p.KeepOutput(object, ans.Output); err != nil { + outcome, said = sdkstep.OutcomeFailed, fmt.Sprintf("the run's answer of %d bytes could not be kept: %v", len(ans.Output), err) + } else { + end.OutputObject = object + } + } if len(said) > 300 { said = said[:300] + "..." } if said == "" && outcome != sdkstep.OutcomePassed { said = "the ask did not pass" } - end := sdkstep.StepEnded{ID: ask.ID, Asker: askerOf(msg, ask), Ask: ask.ID, Node: p.Node, Outcome: outcome, Said: said, At: now().UTC()} + end.Outcome, end.Said = outcome, said body, err := sdkstep.EncodeStepEnded(end) if err != nil { p.logf("the end of the ask %s cannot be said on %s: %v", ask.ID, p.Node, err) @@ -995,3 +1016,19 @@ func (p *Puller) sayEnded(msg *Pulled, ask sdkstep.StepAsk, outcome, said string } p.say("step-ended", json.RawMessage(body), "step-ended."+ask.ID) } + +// runAnswerOf is the run answer a run ask's step handed back, whatever its form (the value or a pointer). +func runAnswerOf(ask sdkstep.StepAsk, result any) (sdkstep.RunAnswer, bool) { + if ask.Run == nil { + return sdkstep.RunAnswer{}, false + } + switch a := result.(type) { + case sdkstep.RunAnswer: + return a, true + case *sdkstep.RunAnswer: + if a != nil { + return *a, true + } + } + return sdkstep.RunAnswer{}, false +} diff --git a/modules/mesh-delivery-sdk/steprun/run_output_test.go b/modules/mesh-delivery-sdk/steprun/run_output_test.go new file mode 100644 index 00000000..240555e6 --- /dev/null +++ b/modules/mesh-delivery-sdk/steprun/run_output_test.go @@ -0,0 +1,83 @@ +package steprun + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + "time" + + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" +) + +// Design 47 G A-254: a build seat's step-ended of a run carries the run's answer, the last line of its program's standard output: +// inline as `output` up to the event bound, else as `output-object`, the object the holder keeps, which the verb `output` answers. +func TestStepEndedOfARunCarriesItsAnswer(t *testing.T) { + big := `"` + strings.Repeat("x", sdkstep.StepEndedInlineMost) + `"` + for _, c := range []struct { + name string + answer sdkstep.RunAnswer + keepErr error + outcome string + inline string + objectKept bool + said string + }{ + {name: "a small answer is inline", answer: sdkstep.RunAnswer{Output: json.RawMessage(`{"verdict":true}`)}, outcome: sdkstep.OutcomePassed, inline: `{"verdict":true}`}, + {name: "an answer over the event bound is an object", answer: sdkstep.RunAnswer{Output: json.RawMessage(big)}, outcome: sdkstep.OutcomePassed, objectKept: true}, + {name: "a program that exits other than 0 failed, its answer kept", answer: sdkstep.RunAnswer{Exit: 3, Output: json.RawMessage(`"no"`), Said: "bad input"}, + outcome: sdkstep.OutcomeFailed, inline: `"no"`, said: "the program exited 3: bad input"}, + {name: "an answer the holder cannot keep is said", answer: sdkstep.RunAnswer{Output: json.RawMessage(big)}, keepErr: errors.New("store full"), + outcome: sdkstep.OutcomeFailed, said: "store full"}, + } { + t.Run(c.name, func(t *testing.T) { + src, st, sy := NewFakeAsks(), newSteps(), &sayer{} + a := testAsk(sdkstep.AskCheck) + a.Check = nil + in, _ := sdkstep.RunInputOf("seat:mesh-catalog.input", []byte(`{"v":1}`)) + commit := "0123456789abcdef0123456789abcdef01234567" + a.Run = &sdkstep.RunJob{Program: sdkstep.RunProgram{Repository: "novox/mesh-delivery", Commit: commit, Dir: "modules/x", Package: "./cmd/x"}, Input: in} + _ = src.PublishAsk(a) + var mu sync.Mutex + kept := map[string][]byte{} + p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, + PullWait: 10 * time.Millisecond, Heartbeat: time.Hour, Say: sy.say, + KeepOutput: func(object string, body []byte) error { + mu.Lock() + defer mu.Unlock() + if c.keepErr != nil { + return c.keepErr + } + kept[object] = body + return nil + }, + Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) { return c.answer, nil }} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Pull(ctx) + waitFor(t, "step-ended", func() bool { return len(sy.of("step-ended")) == 1 }) + end := sy.of("step-ended")[0].body + if end["outcome"] != c.outcome { + t.Fatalf("outcome %v, want %s (%v)", end["outcome"], c.outcome, end) + } + if c.inline != "" && end["output"] != c.inline { + t.Fatalf("output %v, want %s inline", end["output"], c.inline) + } + if c.said != "" && !strings.Contains(end["said"].(string), c.said) { + t.Fatalf("said %v, want it to hold %q", end["said"], c.said) + } + mu.Lock() + defer mu.Unlock() + if c.objectKept { + name := sdkstep.OutputObjectName(a.ID) + if end["output-object"] != name || end["output"] != nil || string(kept[name]) != big { + t.Fatalf("end %v and kept %d bytes, want the object %s holding the whole answer and no inline output", end["output-object"], len(kept[name]), name) + } + } else if end["output-object"] != nil || len(kept) != 0 { + t.Fatalf("an object %v was named or kept (%d), want none", end["output-object"], len(kept)) + } + }) + } +} diff --git a/modules/mesh-delivery-sdk/steprun/seat_tools.go b/modules/mesh-delivery-sdk/steprun/seat_tools.go index 408245c8..1c423783 100644 --- a/modules/mesh-delivery-sdk/steprun/seat_tools.go +++ b/modules/mesh-delivery-sdk/steprun/seat_tools.go @@ -1,6 +1,10 @@ package steprun import ( + "encoding/base64" + "errors" + + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" stdio "git.novox.be/novox/mesh-sdk/go" ) @@ -50,3 +54,29 @@ func SeatTools(seat, what string, p *Puller) []stdio.Tool { }), } } + +// OutputTool serves `.output` (design 47 G A-254, A-257): a run's answer by its ask, as the object the holder keeps it in +// (sdkstep.OutputObjectName), read with read, which answers nil for none. The verb's answer is marked chunks: the bundle answers +// the whole answer as the member `file` (base64) and the tool runner sends it in chunks and answers their count; an ask +// whose answer this node does not keep answers no chunks. +func OutputTool(seat string, read func(object string) ([]byte, error)) stdio.Tool { + return stdio.Tool{Name: seat + ".output", + Description: "A run's answer by its ask, when its step-ended named it as an output-object: the answer's bytes in chunks, of any size.", + Input: map[string]any{"type": "object", "required": []string{"ask", "call"}, "properties": map[string]any{ + "ask": map[string]any{"type": "string", "description": "the ask's id"}, + "call": map[string]any{"type": "string", "description": "the subject token of this send's chunks"}}}, + Run: func(args map[string]any) (any, error) { + ask, _ := args["ask"].(string) + if ask == "" { + return nil, errors.New("the verb output is asked for no ask") + } + body, err := read(sdkstep.OutputObjectName(ask)) + if err != nil { + return nil, err + } + if body == nil { + return map[string]any{"ask": ask, "chunks": 0}, nil + } + return map[string]any{"ask": ask, "file": base64.StdEncoding.EncodeToString(body)}, nil + }} +} diff --git a/modules/mesh-delivery-sdk/steprun/seat_tools_test.go b/modules/mesh-delivery-sdk/steprun/seat_tools_test.go index 74affb11..ddd615da 100644 --- a/modules/mesh-delivery-sdk/steprun/seat_tools_test.go +++ b/modules/mesh-delivery-sdk/steprun/seat_tools_test.go @@ -50,3 +50,27 @@ func TestSeatToolsCurrentPauseResume(t *testing.T) { } waitFor(t, "the ask taken after the resume", func() bool { return src.Waiting(sdkstep.AskDelivery) == 0 }) } + +// Design 47 G A-254: `.output` answers a run's answer by its ask as the whole file (base64), which the tool runner sends in +// chunks (A-257); an ask this node keeps no answer of answers no chunks. +func TestOutputToolAnswersTheKeptAnswer(t *testing.T) { + kept := map[string][]byte{sdkstep.OutputObjectName("a1"): []byte(`{"verdict":"x"}`)} + tool := OutputTool("node-build-go", func(object string) ([]byte, error) { return kept[object], nil }) + if tool.Name != "node-build-go.output" { + t.Fatalf("name %q", tool.Name) + } + got, err := tool.Run(map[string]any{"ask": "a1", "call": "c"}) + if err != nil { + t.Fatal(err) + } + if m := got.(map[string]any); m["ask"] != "a1" || m["file"] != "eyJ2ZXJkaWN0IjoieCJ9" { + t.Fatalf("answer %v, want the ask and its answer's bytes as the file", got) + } + got, _ = tool.Run(map[string]any{"ask": "none", "call": "c"}) + if m := got.(map[string]any); m["ask"] != "none" || m["chunks"] != 0 || m["file"] != nil { + t.Fatalf("answer %v, want no chunks and no file", got) + } + if _, err := tool.Run(map[string]any{"call": "c"}); err == nil { + t.Fatal("a call with no ask was answered") + } +} From 5094abded9d47a49acdc8d57828ce1fa9b340edd Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 09:40:36 +0100 Subject: [PATCH 6/6] C1c-3: the delivery SDK gives askers one reader of a run's answer from a step-ended StepEnded.ReadOutput takes the inline `output`, else asks the seat's verb output on the node that ran the ask for the object `output-object` names (the runtime hands the chunks as the member `file`, A-249); a step-ended with both is refused. Builds: design 47 G A-254, A-240, A-247 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-delivery-sdk/buildstep/stepoutput.go | 60 ++++++++++++++++++ .../buildstep/stepoutput_test.go | 63 +++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 modules/mesh-delivery-sdk/buildstep/stepoutput.go create mode 100644 modules/mesh-delivery-sdk/buildstep/stepoutput_test.go diff --git a/modules/mesh-delivery-sdk/buildstep/stepoutput.go b/modules/mesh-delivery-sdk/buildstep/stepoutput.go new file mode 100644 index 00000000..9cf25f4b --- /dev/null +++ b/modules/mesh-delivery-sdk/buildstep/stepoutput.go @@ -0,0 +1,60 @@ +package buildstep + +import ( + "crypto/rand" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" +) + +// OutputVerb is the verb of a build seat that answers a run's answer kept as an object (design 47 G A-254). +const OutputVerb = "output" + +// AskVerb asks a seat verb as the runtime calls it (stdio.Ask): the key `seat:.[@]` and the body in, the +// verb's answer out. A verb marked chunks is asked by the plain call, and the runtime hands the whole file in `file` (A-249). +type AskVerb func(key string, body any) (json.RawMessage, error) + +// ReadOutput is the answer a build seat's step-ended carries (design 47 G A-254): its inline `output`, else the object +// `output-object` names, read through the verb `output` of the seat of language on the node that ran the ask. Every asker of a +// run reads the answer this way, whichever build seat ran it; none reads the seat's step state. +func (e StepEnded) ReadOutput(language string, ask AskVerb) ([]byte, error) { + if e.OutputObject == "" { + return []byte(e.Output), nil + } + seat, err := BuildSeat(language) + if err != nil { + return nil, err + } + if ask == nil { + return nil, errors.New("buildstep: the answer is in the output object " + e.OutputObject + ", which this reader has no verb to read") + } + var call [16]byte + if _, err := rand.Read(call[:]); err != nil { + return nil, err + } + raw, err := ask("seat:"+seat+"."+OutputVerb+"@"+e.Node, map[string]any{"ask": e.Ask, "call": hex.EncodeToString(call[:])}) + if err != nil { + return nil, fmt.Errorf("buildstep: asking %s for the answer %s of the ask %s: %w", seat, e.OutputObject, e.Ask, err) + } + var got struct { + Ask string `json:"ask"` + File string `json:"file"` + Chunks int `json:"chunks"` + } + if err := json.Unmarshal(raw, &got); err != nil { + return nil, fmt.Errorf("buildstep: the verb output of %s answered no answer for %s: %w", seat, e.Ask, err) + } + if got.Ask != e.Ask { + return nil, fmt.Errorf("buildstep: the verb output of %s answered the ask %q, asked %s", seat, got.Ask, e.Ask) + } + if got.File == "" { + return nil, fmt.Errorf("buildstep: %s on %s keeps no answer of the ask %s", seat, e.Node, e.Ask) + } + body, err := base64.StdEncoding.DecodeString(got.File) + if err != nil { + return nil, fmt.Errorf("buildstep: the answer of %s in no base64: %w", e.Ask, err) + } + return body, nil +} diff --git a/modules/mesh-delivery-sdk/buildstep/stepoutput_test.go b/modules/mesh-delivery-sdk/buildstep/stepoutput_test.go new file mode 100644 index 00000000..d5b6cbb1 --- /dev/null +++ b/modules/mesh-delivery-sdk/buildstep/stepoutput_test.go @@ -0,0 +1,63 @@ +package buildstep + +import ( + "encoding/json" + "errors" + "strings" + "testing" +) + +// Design 47 G A-254: one reader of a run's answer from a step-ended, inline or through the seat's verb output on the node that ran it. +func TestStepEndedReadOutput(t *testing.T) { + const ask = "m2kx9a1b3c4d5f" + var key string + var body map[string]any + verb := func(answer string, err error) AskVerb { + return func(k string, b any) (json.RawMessage, error) { + key = k + body, _ = b.(map[string]any) + return json.RawMessage(answer), err + } + } + for _, c := range []struct { + name string + end StepEnded + ask AskVerb + want string + wantKey string + fail string + }{ + {name: "inline", end: StepEnded{Ask: ask, Node: "ace", Output: `{"a":1}`}, want: `{"a":1}`}, + {name: "an object is read through the verb on its node", end: StepEnded{Ask: ask, Node: "ace", OutputObject: OutputObjectName(ask)}, + ask: verb(`{"ask":"`+ask+`","file":"eyJhIjoxfQ=="}`, nil), want: `{"a":1}`, wantKey: "seat:node-build-go.output@ace"}, + {name: "no answer kept", end: StepEnded{Ask: ask, Node: "ace", OutputObject: "o"}, ask: verb(`{"ask":"`+ask+`","chunks":0}`, nil), fail: "keeps no answer"}, + {name: "another ask's answer", end: StepEnded{Ask: ask, Node: "ace", OutputObject: "o"}, ask: verb(`{"ask":"x","file":"e30="}`, nil), fail: "answered the ask"}, + {name: "the verb fails", end: StepEnded{Ask: ask, Node: "ace", OutputObject: "o"}, ask: verb(``, errors.New("no seat")), fail: "no seat"}, + {name: "no verb", end: StepEnded{Ask: ask, Node: "ace", OutputObject: "o"}, fail: "no verb"}, + } { + t.Run(c.name, func(t *testing.T) { + key, body = "", nil + got, err := c.end.ReadOutput("go", c.ask) + if c.fail != "" { + if err == nil || !strings.Contains(err.Error(), c.fail) { + t.Fatalf("err %v, want %q", err, c.fail) + } + return + } + if err != nil || string(got) != c.want || key != c.wantKey { + t.Fatalf("got %q, %v on %q, want %q on %q", got, err, key, c.want, c.wantKey) + } + if c.wantKey != "" && (body["ask"] != ask || len(body["call"].(string)) != 32) { + t.Fatalf("asked with %v, want the ask and 128 random bits as call", body) + } + }) + } +} + +// A step-ended names its answer inline or as an object, never both. +func TestStepEndedRefusesAnswerTwice(t *testing.T) { + e := StepEnded{ID: "m2kx9a1b3c4d5f", Ask: "m2kx9a1b3c4d5f", Asker: "a", Node: "ace", Outcome: OutcomePassed, Output: "1", OutputObject: "o"} + if _, err := EncodeStepEnded(e); err == nil || !strings.Contains(err.Error(), "inline and as an object") { + t.Fatalf("err %v", err) + } +}