From 604f1528d6b03219faf53bf8d4de7e9e268dac06 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 18:29:56 +0100 Subject: [PATCH] Builder owns the step runner; a build's imports reach its done key and log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildrun leaves mesh-sdk for the builder's own Go module (ADR 0593 §1); the checker, assigner, indexer and watchman require it by a replace to the sibling directory. Issue 1930's bound was already on mesh-sdk's main and moves with it. The builder copies the imports a build module answers to the done key and the log's end entry, where the indexer keeps them (§2; D7). Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- .../cmd/mesh-delivery-assigner/asks.go | 2 +- .../cmd/mesh-delivery-assigner/main.go | 2 +- modules/mesh-delivery-assigner/go.mod | 3 + .../mesh-delivery-builder/buildrun/asks.go | 316 +++++++++++ .../buildrun/asks_integration_test.go | 61 +++ .../buildrun/asks_test.go | 250 +++++++++ .../mesh-delivery-builder/buildrun/choose.go | 88 +++ .../buildrun/choose_test.go | 99 ++++ modules/mesh-delivery-builder/buildrun/doc.go | 5 + modules/mesh-delivery-builder/buildrun/go.mod | 22 + modules/mesh-delivery-builder/buildrun/go.sum | 27 + .../mesh-delivery-builder/buildrun/have.go | 56 ++ .../buildrun/heardends.go | 236 ++++++++ .../mesh-delivery-builder/buildrun/lostask.go | 503 ++++++++++++++++++ .../mesh-delivery-builder/buildrun/modules.go | 7 + .../mesh-delivery-builder/buildrun/places.go | 145 +++++ .../buildrun/places_test.go | 104 ++++ .../buildrun/presence.go | 53 ++ .../buildrun/presence_test.go | 43 ++ .../buildrun/standing.go | 225 ++++++++ .../buildrun/standing_test.go | 263 +++++++++ .../mesh-delivery-builder/buildrun/steps.go | 67 +++ .../buildrun/steps_test.go | 116 ++++ .../buildrun/toolchains.go | 64 +++ .../buildrun/toolchains_test.go | 49 ++ .../cmd/mesh-delivery-builder-tools/main.go | 2 +- .../cmd/mesh-delivery-builder/main.go | 2 +- .../cmd/mesh-delivery-builder/refused_test.go | 2 +- .../cmd/mesh-delivery-builder/toolchains.go | 2 +- modules/mesh-delivery-builder/go.mod | 3 + .../internal/build/askagain.go | 2 +- .../internal/build/buildagain.go | 2 +- .../internal/build/candidates.go | 2 +- .../internal/build/claim.go | 2 +- .../internal/build/imports_test.go | 36 ++ .../internal/build/plan.go | 2 +- .../internal/build/plan_test.go | 2 +- .../internal/build/round4_test.go | 2 +- .../internal/build/run.go | 25 +- .../internal/build/run_test.go | 2 +- .../internal/build/stuck.go | 2 +- .../cmd/mesh-delivery-checker-tools/main.go | 2 +- .../mesh-delivery-checker/integration_test.go | 2 +- .../cmd/mesh-delivery-checker/judge.go | 2 +- .../cmd/mesh-delivery-checker/judge_test.go | 2 +- .../cmd/mesh-delivery-checker/main.go | 2 +- .../cmd/mesh-delivery-checker/refused_test.go | 2 +- .../cmd/mesh-delivery-checker/toolchains.go | 2 +- modules/mesh-delivery-checker/go.mod | 3 + .../internal/check/examiner_test.go | 2 +- .../internal/check/judge.go | 2 +- .../internal/check/lostask.go | 2 +- .../internal/check/lostask_test.go | 2 +- .../internal/check/read.go | 2 +- .../internal/check/try.go | 2 +- .../cmd/mesh-delivery-indexer/main.go | 2 +- modules/mesh-delivery-indexer/go.mod | 3 + modules/mesh-delivery-watchman/go.mod | 3 + 58 files changed, 2902 insertions(+), 31 deletions(-) create mode 100644 modules/mesh-delivery-builder/buildrun/asks.go create mode 100644 modules/mesh-delivery-builder/buildrun/asks_integration_test.go create mode 100644 modules/mesh-delivery-builder/buildrun/asks_test.go create mode 100644 modules/mesh-delivery-builder/buildrun/choose.go create mode 100644 modules/mesh-delivery-builder/buildrun/choose_test.go create mode 100644 modules/mesh-delivery-builder/buildrun/doc.go create mode 100644 modules/mesh-delivery-builder/buildrun/go.mod create mode 100644 modules/mesh-delivery-builder/buildrun/go.sum create mode 100644 modules/mesh-delivery-builder/buildrun/have.go create mode 100644 modules/mesh-delivery-builder/buildrun/heardends.go create mode 100644 modules/mesh-delivery-builder/buildrun/lostask.go create mode 100644 modules/mesh-delivery-builder/buildrun/modules.go create mode 100644 modules/mesh-delivery-builder/buildrun/places.go create mode 100644 modules/mesh-delivery-builder/buildrun/places_test.go create mode 100644 modules/mesh-delivery-builder/buildrun/presence.go create mode 100644 modules/mesh-delivery-builder/buildrun/presence_test.go create mode 100644 modules/mesh-delivery-builder/buildrun/standing.go create mode 100644 modules/mesh-delivery-builder/buildrun/standing_test.go create mode 100644 modules/mesh-delivery-builder/buildrun/steps.go create mode 100644 modules/mesh-delivery-builder/buildrun/steps_test.go create mode 100644 modules/mesh-delivery-builder/buildrun/toolchains.go create mode 100644 modules/mesh-delivery-builder/buildrun/toolchains_test.go create mode 100644 modules/mesh-delivery-builder/internal/build/imports_test.go diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks.go index 2ceefe7..4c4b394 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks.go @@ -7,9 +7,9 @@ import ( "sync" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "mesh-delivery-assigner/internal/assign" ) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 7f29cf9..44846c5 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -22,12 +22,12 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/provisions" diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index f93545d..d0606dc 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -9,6 +9,7 @@ require ( ) require ( + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun v0.0.0-00010101000000-000000000000 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect github.com/klauspost/compress v1.20.0 // indirect @@ -20,3 +21,5 @@ require ( golang.org/x/sys v0.48.0 // indirect golang.org/x/time v0.15.0 // indirect ) + +replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ../mesh-delivery-builder/buildrun diff --git a/modules/mesh-delivery-builder/buildrun/asks.go b/modules/mesh-delivery-builder/buildrun/asks.go new file mode 100644 index 0000000..fdaf305 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/asks.go @@ -0,0 +1,316 @@ +package buildrun + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// **An asker puts an ask on its build seat's stream and waits for its step** (novox/hq ADR 0525 §1, §6; issue 1887 A4, +// A5). It chooses no node: whichever build node pulls the ask writes `.` in its build module's steps state, +// and the asker reads the first such key that is not running. The builder and the checker share it. + +// AskPublisher puts an encoded ask on subject, de-duplicated by its id. +type AskPublisher interface { + PublishAsk(ctx context.Context, subject, id string, data []byte) error +} + +// AskWaiter watches an ask's steps: every value a key `*.` of the build module's steps state takes, running or +// ended, on the channel it answers, until ctx ends. A watch, not a poll. +type AskWaiter interface { + WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) +} + +// Asker puts asks and waits for their steps. +type Asker struct { + Publisher AskPublisher + Waiter AskWaiter + // Bound is how long a running step (a build node took the ask, its key written running) may stand still, with no new phase, + // before the asker stops waiting; a step that keeps moving is waited for up to 3 Bounds from its start (issue 1930); zero waits as long as ctx. An ask waiting in its stream is not bounded: the stream's backlog is no fault + // (ADR 0525 §6, §9), and the bound starts again when another node takes the ask. + Bound time.Duration +} + +// Put puts a on its stream: the seat's accept subject of its language and kind. +func (k Asker) Put(ctx context.Context, a delivery.StepAsk) error { + raw, err := delivery.EncodeStepAsk(a) + if err != nil { + return err + } + subject, err := delivery.AskSubject(a.Language, a.Kind) + if err != nil { + return err + } + if err := k.Publisher.PublishAsk(ctx, subject, a.ID, raw); err != nil { + return fmt.Errorf("putting the %s ask %s: %w", a.Kind, a.ID, err) + } + return nil +} + +// Wait waits for the step of an ask already put (a builder that started again takes over its own asks with it, never +// putting them twice) and answers it when it ended, whatever its state. The bound runs from the step's start on the node +// that took the ask, read from its running key, so an ask that waits in the stream for a free slot waits as long as ctx. +func (k Asker) Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) { + module, err := delivery.BuildModuleOf(a.Language) + if err != nil { + return delivery.Step{}, err + } + watch, cancel := context.WithCancel(ctx) + defer cancel() + steps, err := k.Waiter.WatchAsk(watch, module, a.ID) + if err != nil { + return delivery.Step{}, err + } + var bound, outerCap <-chan time.Time + var timer, capTimer *time.Timer + defer func() { + if timer != nil { + timer.Stop() + } + if capTimer != nil { + capTimer.Stop() + } + }() + running, since, phase, phaseAt := "", time.Time{}, "", time.Time{} + standStill := func() error { + if phase == "" { + return fmt.Errorf("the ask %s's step on %s did not end within %s of its start: it stood still, saying no phase", a.ID, running, k.Bound) + } + return fmt.Errorf("the ask %s's step on %s did not end within %s: it stood still in %q since %s", + a.ID, running, k.Bound, phase, phaseAt.UTC().Format(time.RFC3339)) + } + for { + select { + case <-ctx.Done(): + return delivery.Step{}, ctx.Err() + case <-bound: + return delivery.Step{}, standStill() + case <-outerCap: + return delivery.Step{}, fmt.Errorf("the ask %s's step on %s did not end within %s of its start: it kept moving (last phase %q at %s) but never ended", + a.ID, running, 3*k.Bound, phase, phaseAt.UTC().Format(time.RFC3339)) + case step, ok := <-steps: + if !ok { + return delivery.Step{}, errors.New("the watch of the ask's step ended") + } + if seen := stepSeenOf(ctx); seen != nil { + seen(step) + } + if step.State != delivery.StepRunning { + return step, nil + } + if k.Bound <= 0 { + continue + } + newRun := step.Node != running || !step.Started.Equal(since) + if !newRun && step.Phase == phase && step.PhaseAt.Equal(phaseAt) { + continue + } + // A node took the ask, or took it again after its run stopped (another node, or the same node's build module + // replaced mid-run: a new start), or its step entered a phase. The bound is on standing still: it runs from the + // step's last movement (its phase's start, else its start), so a step that keeps moving is waited for, up to + // three bounds from its start (issue 1930). + running, since, phase, phaseAt = step.Node, step.Started, step.Phase, step.PhaseAt + last := step.Started + if step.PhaseAt.After(last) { + last = step.PhaseAt + } + left := k.Bound + if !last.IsZero() { + left = time.Until(last.Add(k.Bound)) + } + if timer != nil { + timer.Stop() + } + timer = time.NewTimer(max(left, 0)) + bound = timer.C + if newRun { + if capTimer != nil { + capTimer.Stop() + } + outer := 3 * k.Bound + if !step.Started.IsZero() { + outer = time.Until(step.Started.Add(3 * k.Bound)) + } + capTimer = time.NewTimer(max(outer, 0)) + outerCap = capTimer.C + } + } + } +} + +type stepSeenKey struct{} + +// WithStepSeen is ctx carrying seen, which Asker.Wait hands every value the ask's step key takes while it waits: the node +// that took it, each phase it says while running (delivery.Step.Phase), and its end. A try names its asks' progress so +// (issue 1890). seen runs in the waiting goroutine and must not block. +func WithStepSeen(ctx context.Context, seen func(delivery.Step)) context.Context { + return context.WithValue(ctx, stepSeenKey{}, seen) +} + +func stepSeenOf(ctx context.Context) func(delivery.Step) { + seen, _ := ctx.Value(stepSeenKey{}).(func(delivery.Step)) + return seen +} + +// Ask puts a and waits for its step; a step that did not pass is an error naming its node, its state and its reason. +func (k Asker) Ask(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) { + if err := k.Put(ctx, a); err != nil { + return delivery.Step{}, err + } + return k.Passed(k.Wait(ctx, a)) +} + +// Passed is step when it passed, else an error naming where it ended and why. +func (k Asker) Passed(step delivery.Step, err error) (delivery.Step, error) { + if err != nil { + return step, err + } + if step.State != delivery.StepPassed { + return step, fmt.Errorf("the ask %s on %s %s: %s", step.ID, step.Node, step.State, step.Reason) + } + return step, nil +} + +// JetStreamAsks is the bus side of an Asker: a JetStream publish de-duplicated by the ask's id (Nats-Msg-Id), and a watch +// of `*.` in the build module's steps bucket. +type JetStreamAsks struct { + JS jetstream.JetStream +} + +// PublishAsk is AskPublisher's. +func (s JetStreamAsks) PublishAsk(ctx context.Context, subject, id string, data []byte) error { + msg := &nats.Msg{Subject: subject, Data: data, Header: nats.Header{}} + msg.Header.Set(jetstream.MsgIDHeader, id) + _, err := s.JS.PublishMsg(ctx, msg) + return err +} + +// WatchAsk is AskWaiter's: the watch hands over what the keys hold now, so a step that ended before it began is read at +// once, and every change after. +func (s JetStreamAsks) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) { + pattern, err := delivery.AskStepWatch(askID) + if err != nil { + return nil, err + } + kv, err := s.JS.KeyValue(ctx, delivery.StepsBucket(module)) + if err != nil { + return nil, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err) + } + w, err := kv.Watch(ctx, pattern) + if err != nil { + return nil, err + } + out := make(chan delivery.Step) + go func() { + defer close(out) + defer w.Stop() + for { + select { + case <-ctx.Done(): + return + case e, ok := <-w.Updates(): + if !ok { + return + } + if e == nil || e.Operation() != jetstream.KeyValuePut { + continue + } + step, err := delivery.DecodeStep(e.Value()) + if err != nil { + continue // not a step (a node's pause key never matches an ask id; anything else is not this ask's) + } + select { + case out <- step: + case <-ctx.Done(): + return + } + } + } + }() + return out, nil +} + +// FakeAsks is a test double of the bus for an asker's tests: each ask put is kept and, when Answer is set, its step is +// written at once with what Answer says a build node wrote (running or ended). Write writes a step later. Safe for +// concurrent use. +type FakeAsks struct { + Answer func(a delivery.StepAsk) delivery.Step + + mu sync.Mutex + asks []delivery.StepAsk + steps map[string][]delivery.Step + watchers map[string][]chan delivery.Step +} + +// PublishAsk is AskPublisher's. +func (f *FakeAsks) PublishAsk(_ context.Context, _, id string, data []byte) error { + a, err := delivery.DecodeStepAsk(data) + if err != nil { + return err + } + f.mu.Lock() + f.asks = append(f.asks, a) + f.mu.Unlock() + if f.Answer != nil { + f.Write(f.Answer(a)) + } + return nil +} + +// Write writes an ask's step, as a build node does. +func (f *FakeAsks) Write(step delivery.Step) { + f.mu.Lock() + if f.steps == nil { + f.steps, f.watchers = map[string][]delivery.Step{}, map[string][]chan delivery.Step{} + } + f.steps[step.ID] = append(f.steps[step.ID], step) + ws := append([]chan delivery.Step(nil), f.watchers[step.ID]...) + f.mu.Unlock() + for _, w := range ws { + w <- step + } +} + +// End is Write: kept for the tests that end a step. +func (f *FakeAsks) End(step delivery.Step) { f.Write(step) } + +// WatchAsk is AskWaiter's: what was written so far, then every write. +func (f *FakeAsks) WatchAsk(ctx context.Context, _ string, askID string) (<-chan delivery.Step, error) { + out := make(chan delivery.Step, 64) + f.mu.Lock() + if f.steps == nil { + f.steps, f.watchers = map[string][]delivery.Step{}, map[string][]chan delivery.Step{} + } + for _, s := range f.steps[askID] { + out <- s + } + f.watchers[askID] = append(f.watchers[askID], out) + f.mu.Unlock() + go func() { + <-ctx.Done() + f.mu.Lock() + ws := f.watchers[askID] + for i, w := range ws { + if w == out { + f.watchers[askID] = append(ws[:i], ws[i+1:]...) + break + } + } + f.mu.Unlock() + }() + return out, nil +} + +// Asks are the asks put so far. +func (f *FakeAsks) Asks() []delivery.StepAsk { + f.mu.Lock() + defer f.mu.Unlock() + return append([]delivery.StepAsk(nil), f.asks...) +} diff --git a/modules/mesh-delivery-builder/buildrun/asks_integration_test.go b/modules/mesh-delivery-builder/buildrun/asks_integration_test.go new file mode 100644 index 0000000..6c6961c --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/asks_integration_test.go @@ -0,0 +1,61 @@ +//go:build integration + +package buildrun + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +// 1887/A4a, 1887/AC3 on the real bus: an ask put twice under one id is one message, and the asker reads the step a node +// wrote under `.`. +func TestAnAskIsPutOnceAndItsStepIsRead(t *testing.T) { + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, _ := jetstream.New(nc) + ctx := context.Background() + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "SEAT_NODE_BUILD_GO", Subjects: []string{"mesh.seat.node-build-go.accept.>"}, + Retention: jetstream.WorkQueuePolicy}); err != nil { + t.Fatal(err) + } + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket("mesh-build-go")}) + if err != nil { + t.Fatal(err) + } + bus := JetStreamAsks{JS: js} + k := Asker{Publisher: bus, Waiter: bus, Bound: 5 * time.Second} + a := goAsk("mvl0frv18d1960") + for i := 0; i < 2; i++ { + if err := k.Put(ctx, a); err != nil { + t.Fatal(err) + } + } + s, _ := js.Stream(ctx, "SEAT_NODE_BUILD_GO") + if info, _ := s.Info(ctx); info.State.Msgs != 1 { + t.Fatalf("%d messages for one ask put twice", info.State.Msgs) + } + go func() { + time.Sleep(100 * time.Millisecond) + key, _ := delivery.AskStepKey("ace", a.ID) + running, _ := delivery.EncodeStep(delivery.Step{ID: a.ID, Tool: "delivery ask", Node: "ace", State: delivery.StepRunning, Started: time.Now()}) + _, _ = kv.Put(ctx, key, running) + passed, _ := delivery.EncodeStep(delivery.Step{ID: a.ID, Tool: "delivery ask", Node: "ace", State: delivery.StepPassed, Started: time.Now(), + Result: json.RawMessage(`{"artifacts":[]}`)}) + _, _ = kv.Put(ctx, key, passed) + }() + step, err := k.Passed(k.Wait(ctx, a)) + if err != nil || step.Node != "ace" { + t.Fatalf("step %+v, %v", step, err) + } +} diff --git a/modules/mesh-delivery-builder/buildrun/asks_test.go b/modules/mesh-delivery-builder/buildrun/asks_test.go new file mode 100644 index 0000000..5403a54 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/asks_test.go @@ -0,0 +1,250 @@ +package buildrun + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +func goAsk(id string) delivery.StepAsk { + return delivery.StepAsk{ID: id, Kind: delivery.AskDelivery, Language: delivery.AskGo, Asker: "mesh-delivery-builder", At: time.Now(), + Build: &delivery.BuildJob{Module: "m", Path: ".", Fetch: map[string]any{"repository": "novox/x", "commit": strings.Repeat("a", 40)}, + Artifacts: []delivery.AskArtifact{{Artifact: "program"}}}} +} + +// 1887/A4a: an asker puts the ask once and answers the step a build node wrote for it, naming the node; one that did not +// pass is an error naming the node and why; a wait of an ask already put puts nothing. +func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { + f := &FakeAsks{Answer: func(a delivery.StepAsk) delivery.Step { + return delivery.Step{ID: a.ID, Node: "ace", State: delivery.StepPassed, Result: []byte(`{"artifacts":[]}`)} + }} + k := Asker{Publisher: f, Waiter: f} + var seen []delivery.Step // issue 1890: a waiter that asks to see each value of the step sees it + step, err := k.Ask(WithStepSeen(context.Background(), func(s delivery.Step) { seen = append(seen, s) }), goAsk("mvl0frv18d1957")) + if err != nil || step.Node != "ace" || len(f.Asks()) != 1 || len(seen) != 1 || seen[0].Node != "ace" { + t.Fatalf("step %+v, %v, asks %d, seen %+v", step, err, len(f.Asks()), seen) + } + f.Answer = func(a delivery.StepAsk) delivery.Step { + return delivery.Step{ID: a.ID, Node: "shanks", State: delivery.StepRefused, Reason: "go 1.27 installed"} + } + if _, err := k.Ask(context.Background(), goAsk("mvl0frv18d1958")); err == nil || !strings.Contains(err.Error(), "shanks refused: go 1.27 installed") { + t.Fatalf("a refused step: %v", err) + } + // review round 2 item 1: an ask that waits in its stream for a free slot is not bounded; once a node writes its step + // running, the step is bounded from its start, and the error names the node. + g := &FakeAsks{} + k = Asker{Publisher: g, Waiter: g, Bound: 20 * time.Millisecond} + waiting, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if _, err := k.Wait(waiting, goAsk("mvl0frv18d1959")); err != context.DeadlineExceeded { + t.Fatalf("an ask no node took: %v, want it still waiting when ctx ended", err) + } + g.Write(delivery.Step{ID: "mvl0frv18d1961", Node: "ace", State: delivery.StepRunning, Started: time.Now()}) + if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1961")); err == nil || !strings.Contains(err.Error(), "on ace did not end within") { + t.Fatalf("a step that never ends: %v", err) + } + if len(g.Asks()) != 0 { + t.Fatal("a wait put an ask") + } + // Issue 1930: the bound is on a step that stands still, not on its age. A step that enters a new phase inside the bound is + // waited for until it ends; one that stays in a phase the bound fails at once, naming the phase; one that never stops moving + // ends at three bounds, said as such. + k = Asker{Publisher: g, Waiter: g, Bound: 60 * time.Millisecond} + started := time.Now() + go func() { + for i := range 8 { + g.Write(delivery.Step{ID: "mvl0frv18d1980", Node: "ace", State: delivery.StepRunning, Started: started, Phase: "p" + string(rune('a'+i)), PhaseAt: time.Now()}) + time.Sleep(15 * time.Millisecond) + } + g.Write(delivery.Step{ID: "mvl0frv18d1980", Node: "ace", State: delivery.StepPassed}) + }() + if step, err := k.Wait(context.Background(), goAsk("mvl0frv18d1980")); err != nil || step.State != delivery.StepPassed { + t.Fatalf("a step that kept moving past its bound: %+v, %v", step, err) + } + g.Write(delivery.Step{ID: "mvl0frv18d1981", Node: "ace", State: delivery.StepRunning, Started: time.Now(), Phase: "integration: tests", PhaseAt: time.Now()}) + if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1981")); err == nil || !strings.Contains(err.Error(), `stood still in "integration: tests"`) { + t.Fatalf("a step that stood still in a phase: %v", err) + } + loop := make(chan struct{}) + defer close(loop) + started = time.Now() + go func() { + for i := 0; ; i++ { + select { + case <-loop: + return + case <-time.After(20 * time.Millisecond): + g.Write(delivery.Step{ID: "mvl0frv18d1982", Node: "ace", State: delivery.StepRunning, Started: started, Phase: "q" + string(rune('a'+i%26)), PhaseAt: time.Now()}) + } + } + }() + if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1982")); err == nil || !strings.Contains(err.Error(), "kept moving") { + t.Fatalf("a step that never stops moving: %v", err) + } + // Issue 734: an ask the build seat gave up on (DEAD_LETTERS keeps its copy with the ask's id) ends its wait as failed + // with the words, though a node had taken it; under NotTakenBound one no node takes ends at the bound; a step written + // first is answered as the node wrote it. + dead := &DeadLetters{} + lw := LostAskWaiter{Inner: g, Dead: dead} + k = Asker{Publisher: g, Waiter: lw, Bound: time.Minute} + g.Write(delivery.Step{ID: "mvl0frv18d1970", Node: "ace", State: delivery.StepRunning, Started: time.Now()}) + given, ok := GivenUpOf(map[string][]string{"Mesh-Dead-Msg-Id": {"mvl0frv18d1970"}, "Mesh-Dead-Deliveries": {"5"}, + "Mesh-Dead-Stream": {"SEAT_NODE_BUILD_GO"}, "Mesh-Dead-Gave-Up": {"2026-10-26T12:26:40Z"}}) + if !ok { + t.Fatal("a dead letter's headers were not read") + } + go dead.Heard(given) + step, err = k.Wait(context.Background(), goAsk("mvl0frv18d1970")) + if err != nil || step.State != delivery.StepFailed || !IsLost(step) || + !strings.Contains(step.Reason, LostWords+" after 5 deliveries at 2026-10-26T12:26:40Z") { + t.Fatalf("a given-up ask: %+v, %v", step, err) + } + if _, err := k.Passed(step, nil); err == nil || !strings.Contains(err.Error(), "on "+LostNode+" failed: "+LostWords) { + t.Fatalf("a given-up ask's error: %v", err) + } + step, err = k.Wait(NotTakenBound(context.Background(), 20*time.Millisecond), goAsk("mvl0frv18d1971")) + if err != nil || !IsNotTaken(step) || IsLost(step) { + t.Fatalf("an ask no node took: %+v, %v", step, err) + } + g.Write(delivery.Step{ID: "mvl0frv18d1972", Node: "g14", State: delivery.StepPassed}) + if step, err = k.Wait(NotTakenBound(context.Background(), time.Minute), goAsk("mvl0frv18d1972")); err != nil || step.Node != "g14" { + t.Fatalf("a step written: %+v, %v", step, err) + } + // Review round 1 of issue 734: a hearing the bus refuses at start is deaf and says so, never holds the start up, and + // tries again on the next ask waited for; heard, it says so and hands the given-up asks on. + fol := &refusingFollower{refuse: 1} + var mu sync.Mutex + var told []Hearing + hearing := NewDeadLetterHearing(context.Background(), fol, []string{delivery.AskGo}, dead, nil, + func(h Hearing) { mu.Lock(); told = append(told, h); mu.Unlock() }) + hearing.Wait() + if why := hearing.Deaf(); !strings.Contains(why, "SEAT_NODE_BUILD_GO: permissions violation") { + t.Fatalf("a refused hearing says %q", why) + } + k = Asker{Publisher: g, Waiter: LostAskWaiter{Inner: g, Dead: dead, Hearing: hearing}, Bound: time.Minute} + g.Write(delivery.Step{ID: "mvl0frv18d1973", Node: "g14", State: delivery.StepPassed}) + if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1973")); err != nil { + t.Fatal(err) + } + hearing.Wait() + mu.Lock() + if hearing.Deaf() != "" || len(told) != 2 || told[0].Heard || !told[1].Heard || fol.calls() != 2 { + t.Fatalf("after an ask: deaf %q, told %+v, followed %d times", hearing.Deaf(), told, fol.calls()) + } + mu.Unlock() + fol.heard(GivenUp{AskID: "mvl0frv18d1974", Deliveries: 5}) + step, err = k.Wait(context.Background(), goAsk("mvl0frv18d1974")) + if err != nil || !IsLost(step) { + t.Fatalf("an ask heard given up after the hearing came back: %+v, %v", step, err) + } + // The lost check of 2026-10-28: a step whose own watch went deaf is read at the next nudge (a step of its build module + // ended) and answered; a nudge of another build module reads nothing; a watch that closes is opened again. + d := &FakeAsks{} + heard := &HeardEnds{Inner: deafWatch{}, Read: d} + k = Asker{Publisher: d, Waiter: heard, Bound: time.Minute} + d.Write(delivery.Step{ID: "mvl0frv18d1980", Node: "shanks", State: delivery.StepPassed}) + got := make(chan delivery.Step, 1) + go func() { s, _ := k.Wait(context.Background(), goAsk("mvl0frv18d1980")); got <- s }() + for heard.Waiting() == 0 { + time.Sleep(time.Millisecond) + } + heard.Nudge(NodeBuildModule) + select { + case s := <-got: + t.Fatalf("a nudge of another build module answered %+v", s) + case <-time.After(20 * time.Millisecond): + } + heard.Nudge(GoBuildModule) + if s := <-got; s.State != delivery.StepPassed || s.Node != "shanks" { + t.Fatalf("a step its watch never handed over: %+v", s) + } + closing := &HeardEnds{Inner: &closingWatch{inner: d}, Read: d} + if s, err := (Asker{Publisher: d, Waiter: closing}).Wait(context.Background(), goAsk("mvl0frv18d1980")); err != nil || s.State != delivery.StepPassed { + t.Fatalf("a watch that closed once: %+v, %v", s, err) + } + for _, row := range []struct { + h *Hearing + want string + }{{nil, health.StateUnhealthy}, {&told[0], health.StateUnhealthy}, {&told[1], health.StateHealthy}} { + var raw []byte + if row.h != nil { + raw, _ = json.Marshal(row.h) + } + if a := HearingHealth(raw); a.State != row.want || a.Why == "" { + t.Errorf("health of %+v = %+v, want %s", row.h, a, row.want) + } + } +} + +// Review (issue 734 path): an ask taken again by the same node (its build module replaced mid-run) is bounded from the +// new run's start, so its pass is answered after the first run's bound would have ended the wait. +func TestARedeliveryToTheSameNodeRestartsTheBound(t *testing.T) { + f := &FakeAsks{} + k := Asker{Publisher: f, Waiter: f, Bound: 60 * time.Millisecond} + const id = "mvl0frv18d1962" + first := time.Now() + f.Write(delivery.Step{ID: id, Node: "ace", State: delivery.StepRunning, Started: first}) + go func() { + time.Sleep(15 * time.Millisecond) + f.Write(delivery.Step{ID: id, Node: "ace", State: delivery.StepRunning, Started: time.Now()}) + time.Sleep(50 * time.Millisecond) // past the first run's bound, inside the second's + f.Write(delivery.Step{ID: id, Node: "ace", State: delivery.StepPassed, Started: time.Now()}) + }() + step, err := k.Wait(context.Background(), goAsk(id)) + if err != nil || step.State != delivery.StepPassed { + t.Fatalf("step %+v, %v, want the redelivered run's pass", step, err) + } +} + +// refusingFollower refuses the first `refuse` follows, as the bus does before the asker's grant is composed. +type refusingFollower struct { + mu sync.Mutex + refuse int + n int + hand func(GivenUp) +} + +func (f *refusingFollower) Follow(_ context.Context, _ string, _ time.Time, heard func(GivenUp)) (func(), error) { + f.mu.Lock() + defer f.mu.Unlock() + f.n++ + if f.n <= f.refuse { + return nil, errors.New("permissions violation") + } + f.hand = heard + return func() {}, nil +} + +func (f *refusingFollower) calls() int { f.mu.Lock(); defer f.mu.Unlock(); return f.n } + +func (f *refusingFollower) heard(g GivenUp) { f.mu.Lock(); h := f.hand; f.mu.Unlock(); h(g) } + +// deafWatch is a watch of an ask's step that stays open and hears nothing. +type deafWatch struct{} + +func (deafWatch) WatchAsk(context.Context, string, string) (<-chan delivery.Step, error) { + return make(chan delivery.Step), nil +} + +// closingWatch closes its first watch at once, and hands the later ones to inner. +type closingWatch struct { + inner AskWaiter + closed bool +} + +func (c *closingWatch) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) { + if !c.closed { + c.closed = true + ch := make(chan delivery.Step) + close(ch) + return ch, nil + } + return c.inner.WatchAsk(ctx, module, askID) +} diff --git a/modules/mesh-delivery-builder/buildrun/choose.go b/modules/mesh-delivery-builder/buildrun/choose.go new file mode 100644 index 0000000..06aea83 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/choose.go @@ -0,0 +1,88 @@ +package buildrun + +import ( + "fmt" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// ChooseNodes picks up to want build nodes for a language (ADR 0478 clause 3): the nodes whose toolchains key reports the +// version the nodes agree on, the ones with room under their build capacity first (places), then the fewest lanes the +// part has running or chosen there, a tie broken by the node's name. A place is reserved on each node chosen, in the +// same step as the choice, so builds choosing at once do not choose the same full node while another has room; the +// caller takes the place (Places.Acquire) or gives it up (Places.Cancel). A node at its capacity is still chosen when +// the others are too (fewer than want have room): its lane waits for a place. A node qualifies only when qualifies is true +// of it (online and its build module healthy; nil means every node does). It answers the toolchain all the reporting nodes +// agree on, qualifying or not, as the indexer computes it. When nothing is reported, or the nodes report different +// versions (so the indexer agreed none), it fails naming the language and why; when no reporting node qualifies it fails +// naming the nodes. A node whose build module refuses on its toolchain refuses the step, and the build fails naming it. +func ChooseNodes(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, delivery.Toolchain2, error) { + tc, none := delivery.AgreedToolchain(entries) + if none != "" { + return nil, delivery.Toolchain2{}, fmt.Errorf("no %s build node qualifies: %s", language, none) + } + nodes, err := choose(language, entries, places, qualifies, want) + return nodes, tc, err +} + +// ChooseAnyNodes is ChooseNodes for artifacts that need no toolchain, such as an archive (ADR 0478 clause 1): the +// build module's nodes agree on nothing, so any node of the build module that qualifies is chosen, by capacity and load only. +func ChooseAnyNodes(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, error) { + return choose(language, entries, places, qualifies, want) +} + +func choose(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, error) { + if places == nil { + places = &Places{} + } + nodes := make([]string, 0, len(entries)) + var reporting []string + seen := map[string]bool{} + for _, e := range entries { + if seen[e.Node] { + continue + } + seen[e.Node] = true + reporting = append(reporting, e.Node) + if places.CapacityOf(e.Node) > 0 && (qualifies == nil || qualifies(e.Node)) { // a capacity of 0 keeps the part off a node + nodes = append(nodes, e.Node) + } + } + if len(nodes) == 0 { + sort.Strings(reporting) + return nil, fmt.Errorf("no %s build node qualifies: %s report the toolchain, and none is online with a healthy build module and a build capacity", language, strings.Join(reporting, ", ")) + } + return places.choose(nodes, want), nil +} + +// DigestsDifferPrefix opens the error of a build whose nodes built an artifact to different digests (ADR 0478 clause 4): +// such a key is not built again as a new revision, since the same source on the same toolchains gave two answers and +// building again settles nothing; it waits for another fingerprint. +const DigestsDifferPrefix = "digests differ:" + +// Agree is the one digest every node built an artifact to (ADR 0478 clause 4), or an error naming each node and its +// digest when they differ. One node's digest agrees with itself. +func Agree(artifact string, byNode map[string]string) (string, error) { + nodes := make([]string, 0, len(byNode)) + for n := range byNode { + nodes = append(nodes, n) + } + sort.Strings(nodes) + var first string + for i, n := range nodes { + if i == 0 { + first = byNode[n] + continue + } + if byNode[n] != first { + msg := fmt.Sprintf("%s the nodes built %s to different digests:", DigestsDifferPrefix, artifact) + for _, n := range nodes { + msg += fmt.Sprintf(" %s %s;", n, byNode[n]) + } + return "", fmt.Errorf("%s", msg[:len(msg)-1]) + } + } + return first, nil +} diff --git a/modules/mesh-delivery-builder/buildrun/choose_test.go b/modules/mesh-delivery-builder/buildrun/choose_test.go new file mode 100644 index 0000000..6e9c2bb --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/choose_test.go @@ -0,0 +1,99 @@ +package buildrun + +import ( + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +const digest = "sha256:2222222222222222222222222222222222222222222222222222222222222222" + +func hex64(c byte) string { return strings.Repeat(string(c), 64) } + +func tcs(language, version string, nodes ...string) []delivery.NodeToolchain { + var out []delivery.NodeToolchain + for _, n := range nodes { + out = append(out, delivery.NodeToolchain{Node: n, Toolchain: delivery.Toolchain{Language: language, Version: version, Recipe: "1"}}) + } + return out +} + +func TestChooseTakesTheAgreedNodesWithFewestRunningStepsTiesByName(t *testing.T) { // 1607/AC4, ADR 0478 clause 3 + entries := tcs("go", "go1.27.1", "novox", "ace", "g14", "shanks") + nodes, tc, err := ChooseNodes("go", entries, placesRunning(map[string]int{"novox": 3, "ace": 1, "g14": 1, "shanks": 0}), nil, 2) + if err != nil { + t.Fatal(err) + } + if len(nodes) != 2 || nodes[0] != "shanks" || nodes[1] != "ace" { + t.Errorf("nodes = %v, want [shanks ace]: fewest steps first, a tie by name", nodes) + } + if tc.Version != "go1.27.1" || tc.Recipe != "1" || tc.Language != "go" { + t.Errorf("toolchain = %+v", tc) + } +} + +func TestChooseWithOneNodeAnswersOneNode(t *testing.T) { // 1607/AC4 + nodes, _, err := ChooseNodes("go", tcs("go", "go1.27.1", "novox"), nil, nil, 2) + if err != nil || len(nodes) != 1 || nodes[0] != "novox" { + t.Errorf("nodes = %v, err %v, want [novox]", nodes, err) + } +} + +func TestChooseFailsNamingTheLanguageAndTheVersionsWhenNoNodeQualifies(t *testing.T) { // 1607/AC4 + _, _, err := ChooseNodes("typescript", nil, nil, nil, 2) + if err == nil || !strings.Contains(err.Error(), "typescript") || !strings.Contains(err.Error(), delivery.ToolchainNoReport) { + t.Errorf("no report: error = %v, want it to name typescript and %q", err, delivery.ToolchainNoReport) + } + entries := append(tcs("go", "go1.27.1", "novox"), tcs("go", "go1.26.0", "ace")...) + _, _, err = ChooseNodes("go", entries, nil, nil, 2) + if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), "go1.26.0") || !strings.Contains(err.Error(), "go") { + t.Errorf("disagreeing nodes: error = %v, want both versions named", err) + } +} + +func TestAgreeNeedsEqualDigestsAndNamesBothNodesWhenTheyDiffer(t *testing.T) { // 1607/AC4, ADR 0478 clause 4 + d, err := Agree("tool", map[string]string{"ace": digest, "novox": digest}) + if err != nil || d != digest { + t.Errorf("Agree = %q, %v, want %q", d, err, digest) + } + _, err = Agree("tool", map[string]string{"ace": digest, "novox": "sha256:" + hex64('9')}) + if err == nil || !strings.Contains(err.Error(), "ace") || !strings.Contains(err.Error(), "novox") || !strings.Contains(err.Error(), digest) || !strings.Contains(err.Error(), hex64('9')) || !strings.Contains(err.Error(), "tool") { + t.Errorf("differing digests: error = %v, want the artifact, both nodes and both digests", err) + } + if d, err := Agree("tool", map[string]string{"ace": digest}); err != nil || d != digest { + t.Errorf("one node: %q, %v", d, err) + } +} + +func TestChooseSkipsNodesThatAreNotOnlineAndFailsNamingThemWhenNoneIs(t *testing.T) { // 1607 review 3, ADR 0478 clause 3 + entries := tcs("go", "go1.27.1", "ace", "g14", "novox") + online := func(n string) bool { return n != "g14" } // a sleeping laptop + nodes, tc, err := ChooseNodes("go", entries, placesRunning(map[string]int{"novox": 5}), online, 2) + if err != nil || len(nodes) != 2 || nodes[0] != "ace" || nodes[1] != "novox" || tc.Version != "go1.27.1" { + t.Errorf("nodes = %v, toolchain %+v, err %v, want [ace novox] with g14 left out although it runs nothing", nodes, tc, err) + } + _, tc, err = ChooseNodes("go", entries, nil, func(string) bool { return false }, 2) + if err == nil || !strings.Contains(err.Error(), "online") || !strings.Contains(err.Error(), "g14") || !strings.Contains(err.Error(), "go") { + t.Errorf("none online: error = %v, want it to name the nodes that report and that none is online", err) + } + if tc.Version != "go1.27.1" { + t.Errorf("none online: the agreed toolchain %+v is still answered", tc) + } +} + +// placesRunning is Places with these steps counted as running, for a test of the choice. +func placesRunning(running map[string]int) *Places { + return &Places{running: running, changed: make(chan struct{})} +} + +func TestChoosePrefersNodesBelowTheirCapacity(t *testing.T) { // round 3 + entries := tcs("go", "go1.27.1", "ace", "novox", "shanks") + // shanks has capacity 3 with 2 running: room; ace has the default 1 with 1 running: none; novox has none running. + p := placesRunning(map[string]int{"shanks": 2, "ace": 1}) + p.Capacity = map[string]int{"shanks": 3} + nodes, _, err := ChooseNodes("go", entries, p, nil, 2) + if err != nil || len(nodes) != 2 || nodes[0] != "novox" || nodes[1] != "shanks" { + t.Errorf("nodes = %v, err %v, want [novox shanks]: the nodes with room, ace at capacity left out", nodes, err) + } +} diff --git a/modules/mesh-delivery-builder/buildrun/doc.go b/modules/mesh-delivery-builder/buildrun/doc.go new file mode 100644 index 0000000..b4a943d --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/doc.go @@ -0,0 +1,5 @@ +// Package buildrun holds the parts every module that runs steps on the build modules shares: the caller, the steps +// watch and the runner of a long step (ADR 0469), the sets of presence, toolchains and healthy build modules, the +// places a node offers by capacity, and the choice of build nodes (ADR 0478). The builder and the checker of the +// delivery use them; they moved here from the builder's own package so neither keeps a copy (issue 1671, ADR 0484). +package buildrun diff --git a/modules/mesh-delivery-builder/buildrun/go.mod b/modules/mesh-delivery-builder/buildrun/go.mod new file mode 100644 index 0000000..7d5675a --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/go.mod @@ -0,0 +1,22 @@ +module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-delivery-builder/buildrun/go.sum b/modules/mesh-delivery-builder/buildrun/go.sum new file mode 100644 index 0000000..ebd4e0f --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-delivery-builder/buildrun/have.go b/modules/mesh-delivery-builder/buildrun/have.go new file mode 100644 index 0000000..f3c079c --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/have.go @@ -0,0 +1,56 @@ +package buildrun + +import ( + "sync" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// HaveSet holds what each node's node-engine says of each module on it (node-engine_have, key `.`): the +// builder reads it to know whether a node's build module is running and healthy, since a build module whose installed +// toolchain is not the one its recipe builds with is unhealthy (mesh-build-go build_go_health) and its steps would +// refuse (ADR 0478 clause 3). A module the node-engine has written no key of is not healthy. Safe for use from the watch +// and from a build at once. +type HaveSet struct { + mu sync.Mutex + healthy map[string]bool // by `.` +} + +// Put takes a change of the have bucket in and answers whether a module's healthy state changed. A node's own key +// (`.node`) and a value that is not a have are left out. +func (s *HaveSet) Put(key string, value []byte, deleted bool) bool { + node, module, isNode, err := delivery.ParseHaveKey(key) + if err != nil || isNode { + return false + } + ok := false + if !deleted { + h, err := delivery.DecodeHave(value) + if err != nil { + return false + } + ok = h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK + } + k := node + "." + module + s.mu.Lock() + defer s.mu.Unlock() + if s.healthy[k] == ok { + return false + } + if s.healthy == nil { + s.healthy = map[string]bool{} + } + if ok { + s.healthy[k] = true + } else { + delete(s.healthy, k) + } + return true +} + +// Healthy is whether the module on the node was last seen running and healthy. +func (s *HaveSet) Healthy(node, module string) bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.healthy[node+"."+module] +} diff --git a/modules/mesh-delivery-builder/buildrun/heardends.go b/modules/mesh-delivery-builder/buildrun/heardends.go new file mode 100644 index 0000000..25b8cbb --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/heardends.go @@ -0,0 +1,236 @@ +package buildrun + +import ( + "context" + "fmt" + "sync" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// **A step's end is never only its own watch's to hear** (the lost check of 2026-10-28: mesh-catalog #553's verdict +// stayed running after its step mvsgthyjed5786 had passed on shanks at 19:09:31Z; the watch of that one key, opened at +// 18:46:05Z, heard neither the step's start nor its end and never closed, while a watch opened at 18:58:48Z heard its own +// step end). HeardEnds stands between the asker (Asker) and the watch of each ask's step key: when it is +// nudged, every ask it waits on reads its step keys now (a read, not the watch) and hands what it reads to its wait, so a +// step that ended is judged even when its own watch went deaf. A watch that closes under a wait is opened again, at most +// reopenTries times. It is nudged on an event (Nudge): any step of the same build module ending (one watch of each steps +// state, HearEnds), a reconnect to the bus, and whatever else the asker hears (the checker: checker_ask_again). +// It holds no timer. Every asker of the build seats waits through it (the checker and the builder, HearEnds). + +// StepReader reads, now, every step a build module's steps state holds for an ask: its keys `*.`. +type StepReader interface { + AskSteps(ctx context.Context, module, askID string) ([]delivery.Step, error) +} + +// reopenTries is how often a wait opens its ask's watch again after it closed, before the wait ends failed. +const reopenTries = 3 + +// HeardEnds is an AskWaiter over Inner that also reads an ask's steps when nudged. Safe for concurrent use. +type HeardEnds struct { + Inner AskWaiter + Read StepReader + Logf func(format string, args ...any) + + mu sync.Mutex + waiting map[string]*waitedAsk // by ask id +} + +type waitedAsk struct { + module string + nudge chan struct{} +} + +func (h *HeardEnds) logf(format string, args ...any) { + if h.Logf != nil { + h.Logf(format, args...) + } +} + +// WatchAsk is AskWaiter's: what Inner's watch hands, and, at each nudge, what the ask's keys hold now. +func (h *HeardEnds) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) { + in, err := h.Inner.WatchAsk(ctx, module, askID) + if err != nil { + return nil, err + } + w := &waitedAsk{module: module, nudge: make(chan struct{}, 1)} + h.mu.Lock() + if h.waiting == nil { + h.waiting = map[string]*waitedAsk{} + } + h.waiting[askID] = w + h.mu.Unlock() + out := make(chan delivery.Step) + go func() { + defer close(out) + defer func() { + h.mu.Lock() + if h.waiting[askID] == w { + delete(h.waiting, askID) + } + h.mu.Unlock() + }() + send := func(s delivery.Step) bool { + select { + case out <- s: + return true + case <-ctx.Done(): + return false + } + } + reopened := 0 + for { + select { + case <-ctx.Done(): + return + case s, ok := <-in: + if !ok { + if ctx.Err() != nil || reopened >= reopenTries { + h.logf("the watch of the ask %s's step on %s closed %d times; its wait ends", askID, module, reopened+1) + return + } + reopened++ + if in, err = h.Inner.WatchAsk(ctx, module, askID); err != nil { + h.logf("the watch of the ask %s's step on %s closed and cannot be opened again, so its wait ends: %v", askID, module, err) + return + } + h.logf("the watch of the ask %s's step on %s closed; opened again (%d of %d)", askID, module, reopened, reopenTries) + continue + } + if !send(s) { + return + } + case <-w.nudge: + if h.Read == nil { + continue + } + steps, err := h.Read.AskSteps(ctx, module, askID) + if err != nil { + h.logf("the steps of the ask %s on %s cannot be read: %v", askID, module, err) + continue + } + for _, s := range steps { + if s.State != delivery.StepRunning { + h.logf("the ask %s's step on %s ended %s, read at a nudge: its watch had not handed it over", askID, s.Node, s.State) + } + if !send(s) { + return + } + } + } + } + }() + return out, nil +} + +// Nudge has every ask waited on of module read its steps now; module "" nudges every ask. +func (h *HeardEnds) Nudge(module string) { + h.mu.Lock() + defer h.mu.Unlock() + for _, w := range h.waiting { + if module == "" || w.module == module { + select { + case w.nudge <- struct{}{}: + default: // a nudge is pending already + } + } + } +} + +// Waiting is how many asks are waited on now. +func (h *HeardEnds) Waiting() int { + h.mu.Lock() + defer h.mu.Unlock() + return len(h.waiting) +} + +// AskSteps is StepReader's: the keys `*.` of the build module's steps state, read now. +func (s JetStreamAsks) AskSteps(ctx context.Context, module, askID string) ([]delivery.Step, error) { + pattern, err := delivery.AskStepWatch(askID) + if err != nil { + return nil, err + } + kv, err := s.JS.KeyValue(ctx, delivery.StepsBucket(module)) + if err != nil { + return nil, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err) + } + lister, err := kv.ListKeysFiltered(ctx, pattern) + if err != nil { + return nil, err + } + var out []delivery.Step + for key := range lister.Keys() { + e, err := kv.Get(ctx, key) + if err != nil { + continue // deleted between the list and the read + } + if step, err := delivery.DecodeStep(e.Value()); err == nil { + out = append(out, step) + } + } + return out, nil +} + +// HearEnds is the HeardEnds every asker of the build seats waits through: over inner (the asks' own watches, or a +// LostAskWaiter's inner), reading with asks, and nudged at each step end of each of modules (one watch of each steps state, +// updates only; one that closes is opened again, and said and left after reopenTries closes in a row that heard nothing). The asker nudges it at a reconnect too (Nudge("")). +func HearEnds(ctx context.Context, asks JetStreamAsks, inner AskWaiter, modules []string, logf func(string, ...any)) *HeardEnds { + if logf == nil { + logf = func(string, ...any) {} + } + h := &HeardEnds{Inner: inner, Read: asks, Logf: logf} + for _, module := range modules { + kv, err := asks.JS.KeyValue(ctx, delivery.StepsBucket(module)) + if err != nil { + logf("the steps of %s cannot be read (%v): an ask whose own watch goes deaf is read only at a reconnect", module, err) + continue + } + go h.follow(ctx, kv, module) + } + return h +} + +func (h *HeardEnds) follow(ctx context.Context, kv jetstream.KeyValue, module string) { + silent := 0 // closes in a row having heard nothing + for ctx.Err() == nil { + w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) + if err != nil { + h.logf("the steps of %s cannot be watched, so an ask whose own watch goes deaf is read only at a reconnect: %v", module, err) + return + } + heard := false + for e := range w.Updates() { + if e == nil || e.Operation() != jetstream.KeyValuePut { + continue + } + heard = true + if step, err := delivery.DecodeStep(e.Value()); err == nil && step.State != delivery.StepRunning { + h.Nudge(module) + } + } + _ = w.Stop() + if ctx.Err() != nil { + return + } + h.Nudge(module) // a step may have ended while the watch was closing + if heard { + silent = 0 + } else if silent++; silent >= reopenTries { + h.logf("the watch of the steps of %s closed %d times in a row having heard nothing, so it is left: an ask whose own watch goes deaf is read only at a reconnect", module, silent) + return + } + h.logf("the watch of the steps of %s closed; opened again", module) + } +} + +// AskSteps is StepReader's: the last step written for the ask, if any. +func (f *FakeAsks) AskSteps(_ context.Context, _ string, askID string) ([]delivery.Step, error) { + f.mu.Lock() + defer f.mu.Unlock() + if s := f.steps[askID]; len(s) > 0 { + return []delivery.Step{s[len(s)-1]}, nil + } + return nil, nil +} diff --git a/modules/mesh-delivery-builder/buildrun/lostask.go b/modules/mesh-delivery-builder/buildrun/lostask.go new file mode 100644 index 0000000..750acaa --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/lostask.go @@ -0,0 +1,503 @@ +package buildrun + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// **An ask the build seat gave up on ends its step** (novox/hq issue 734). A build seat's worker offers an ask +// delivery.AskMaxDeliver times; one handed out that often without an answer (a bus reload that drops every hand-out, a node +// that dies taking it each time) is given up by the server, and the controller keeps a copy of it in DEAD_LETTERS under +// mesh.events.dead.., with the ask's message id (its ask id), how often it was handed out and when it +// was given up. No build node writes a step for it, so an asker that waits for the step waits for ever: a try stays +// running and holds a try place, a check's verdict stays running and its pull request waits. +// +// An asker of the build seats (the checker, the builder) hears those copies (DeadLetterHearing: an ordered consumer on +// DEAD_LETTERS filtered to the build seats' streams, granted to a user of a seat pulled by kind, buscompose +// DeadLetterReads) into DeadLetters, and LostAskWaiter ends the waiting step at once as failed on the node LostNode, its +// reason saying LostWords; the asker decides what follows (the checker ends a try as error and asks a check again, the +// builder asks its part of a build again). +// +// Belt and braces, where the asker says so in the context (NotTakenBound; the checker's tries): an ask no build node took +// within a bound ends the same way, saying NotTakenWords. An ask waits in the stream as long as the backlog is otherwise +// (ADR 0525 §9): a backlog is no fault. + +// LostNode is the node a step ended for a lost ask names: no build node wrote it. +const LostNode = "bus" + +// The words of a step ended for its ask. +const ( + LostWords = "its ask was given up by the build seat" + NotTakenWords = "no build node took its ask" +) + +// The headers the controller keeps a given-up message's copy with in DEAD_LETTERS (mesh-controller internal/link +// deadletters.go): the original message id (an ask's is its ask id), how often it was handed out, when it was given up. +const ( + DeadHeaderMsgID = "Mesh-Dead-Msg-Id" + DeadHeaderDeliveries = "Mesh-Dead-Deliveries" + DeadHeaderGaveUp = "Mesh-Dead-Gave-Up" + DeadHeaderStream = "Mesh-Dead-Stream" +) + +// DeadLettersStream is the stream the controller keeps given-up messages in, and DeadLetterFilter the subjects of one seat +// stream's copies there (buscompose DeadLetterSubject). +const DeadLettersStream = "DEAD_LETTERS" + +// DeadLetterFilter is the subjects DEAD_LETTERS keeps a seat stream's given-up messages under. +func DeadLetterFilter(stream string) string { return "mesh.events.dead." + stream + ".>" } + +// SeatStream is the stream of a build seat's language (buscompose seatStreamName of node-build-). +func SeatStream(language string) string { return "SEAT_NODE_BUILD_" + strings.ToUpper(language) } + +// GivenUp is one ask a build seat gave up on, as DEAD_LETTERS keeps it. +type GivenUp struct { + AskID string + Stream string + Deliveries int + At time.Time +} + +// GivenUpOf reads a DEAD_LETTERS copy's headers; false when it names no message id. +func GivenUpOf(header map[string][]string) (GivenUp, bool) { + get := func(k string) string { + if v := header[k]; len(v) > 0 { + return strings.TrimSpace(v[0]) + } + return "" + } + g := GivenUp{AskID: get(DeadHeaderMsgID), Stream: get(DeadHeaderStream)} + if g.AskID == "" { + return GivenUp{}, false + } + g.Deliveries, _ = strconv.Atoi(get(DeadHeaderDeliveries)) + g.At, _ = time.Parse(time.RFC3339Nano, get(DeadHeaderGaveUp)) + return g, true +} + +// Words are what a step ended for the given-up ask says. +func (g GivenUp) Words() string { + at := "at an unknown time" + if !g.At.IsZero() { + at = "at " + g.At.UTC().Format(time.RFC3339) + } + return fmt.Sprintf("%s after %d deliveries %s (kept in %s); ask again", LostWords, g.Deliveries, at, DeadLettersStream) +} + +// keptGivenUp is how many given-up asks DeadLetters remembers for a waiter that starts after the copy was heard. +const keptGivenUp = 1024 + +// DeadLetters holds the given-up asks heard, and hands each to the waiter of its ask. Safe for concurrent use. +type DeadLetters struct { + mu sync.Mutex + heard map[string]GivenUp + order []string + watchers map[string][]chan GivenUp +} + +// Heard takes one given-up ask in. +func (d *DeadLetters) Heard(g GivenUp) { + d.mu.Lock() + if d.heard == nil { + d.heard, d.watchers = map[string]GivenUp{}, map[string][]chan GivenUp{} + } + if _, ok := d.heard[g.AskID]; !ok { + d.order = append(d.order, g.AskID) + if len(d.order) > keptGivenUp { + delete(d.heard, d.order[0]) + d.order = d.order[1:] + } + } + d.heard[g.AskID] = g + ws := d.watchers[g.AskID] + delete(d.watchers, g.AskID) + d.mu.Unlock() + for _, w := range ws { + w <- g // buffered, one each + } +} + +// watch answers a channel that gets the ask's given-up copy once (at once when it was heard already), and its cancel. +func (d *DeadLetters) watch(askID string) (<-chan GivenUp, func()) { + ch := make(chan GivenUp, 1) + d.mu.Lock() + defer d.mu.Unlock() + if d.heard == nil { + d.heard, d.watchers = map[string]GivenUp{}, map[string][]chan GivenUp{} + } + if g, ok := d.heard[askID]; ok { + ch <- g + return ch, func() {} + } + d.watchers[askID] = append(d.watchers[askID], ch) + return ch, func() { + d.mu.Lock() + defer d.mu.Unlock() + ws := d.watchers[askID] + for i, w := range ws { + if w == ch { + d.watchers[askID] = append(ws[:i], ws[i+1:]...) + break + } + } + if len(d.watchers[askID]) == 0 { + delete(d.watchers, askID) + } + } +} + +type notTakenKey struct{} + +// NotTakenBound says, in ctx, that an ask no build node took within bound ends its step (a try's, never a check's). +func NotTakenBound(ctx context.Context, bound time.Duration) context.Context { + return context.WithValue(ctx, notTakenKey{}, bound) +} + +func notTakenBoundOf(ctx context.Context) time.Duration { + d, _ := ctx.Value(notTakenKey{}).(time.Duration) + return d +} + +// LostAskWaiter is an AskWaiter that ends an ask's step when the build seat gave the ask up (DeadLetters), or, under +// NotTakenBound, when no build node took it within the bound: it hands the asker a failed step whose reason says why, on +// the node "bus". Every step the build module writes is handed on as it is. +type LostAskWaiter struct { + Inner AskWaiter + Dead *DeadLetters + // Hearing, when set, is asked to hear DEAD_LETTERS again each time an ask is waited for while it cannot: an ask put is + // the event that makes hearing matter. + Hearing interface { + Deaf() string + Again() + } + Now func() time.Time +} + +func (w LostAskWaiter) now() time.Time { + if w.Now != nil { + return w.Now() + } + return time.Now() +} + +// WatchAsk is AskWaiter's. +func (w LostAskWaiter) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) { + if w.Hearing != nil && w.Hearing.Deaf() != "" { + w.Hearing.Again() + } + inner, err := w.Inner.WatchAsk(ctx, module, askID) + if err != nil { + return nil, err + } + var dead <-chan GivenUp + cancel := func() {} + if w.Dead != nil { + dead, cancel = w.Dead.watch(askID) + } + // The not-taken bound is a context's (ADR 0437 §8: no timer), ended when a node takes the ask. + var notTaken <-chan struct{} + bound := notTakenBoundOf(ctx) + stopBound := func() {} + if bound > 0 { + var nt context.Context + nt, stopBound = context.WithTimeout(context.Background(), bound) + notTaken = nt.Done() + } + out := make(chan delivery.Step) + go func() { + defer close(out) + defer cancel() + defer func() { stopBound() }() + send := func(s delivery.Step) bool { + select { + case out <- s: + return true + case <-ctx.Done(): + return false + } + } + ended := func(reason string) delivery.Step { + at := w.now().UTC() + return delivery.Step{ID: askID, Node: LostNode, State: delivery.StepFailed, Reason: reason, Ended: at} + } + for { + select { + case <-ctx.Done(): + return + case s, ok := <-inner: + if !ok { + return + } + if notTaken != nil { + stopBound() // a node took it: its step is bounded by the asker from its start + notTaken = nil + } + if !send(s) { + return + } + case g := <-dead: + dead = nil + send(ended(g.Words())) + return + case <-notTaken: + send(ended(fmt.Sprintf("%s within %s of its put; it was lost or waits behind a backlog; ask again", NotTakenWords, bound))) + return + } + } + }() + return out, nil +} + +// IsLost says whether a step was ended by LostAskWaiter because the build seat gave its ask up (LostWords). +func IsLost(s delivery.Step) bool { return s.Node == LostNode && strings.Contains(s.Reason, LostWords) } + +// IsNotTaken says whether a step was ended by LostAskWaiter because no build node took its ask within the bound. +func IsNotTaken(s delivery.Step) bool { + return s.Node == LostNode && strings.Contains(s.Reason, NotTakenWords) +} + +// **The asker hears DEAD_LETTERS, tries again when it cannot, and says whether it hears** (issue 734). The read is +// granted with the asker's seat traffic (buscompose DeadLetterReads); an asker started before its grant was composed is +// refused, and an asker that only tried once would stay deaf until it starts again, silently. DeadLetterHearing tries in +// the background, never holding the start up, and again on an event (Again): the asker's reconnect to the bus, and every +// ask waited for while it is deaf (LostAskWaiter.Hearing); no timer. Each change of whether it hears is said (Said), so +// the module's health tool says it: a deaf asker is not healthy. + +// Hearing is what a DeadLetterHearing says: whether it hears every build seat stream it follows, and why not. +type Hearing struct { + Heard bool `json:"heard"` + Streams []string `json:"streams"` + Why string `json:"why,omitempty"` + At time.Time `json:"at"` +} + +// HearingKey is the key a module keeps its Hearing under, in its own state (the checker's and the builder's hearing). +const HearingKey = "dead-letters" + +// DeadLetterFollower starts following one seat stream's copies in DEAD_LETTERS from since, handing each given-up ask to +// heard; it answers the stop, or why it cannot. +type DeadLetterFollower interface { + Follow(ctx context.Context, stream string, since time.Time, heard func(GivenUp)) (func(), error) +} + +// JetStreamDeadLetters is the bus side of a DeadLetterFollower: an ordered consumer on DEAD_LETTERS filtered to the +// stream's copies, which resumes where it was after a reconnect and, reset before it heard any, starts at since again. +type JetStreamDeadLetters struct{ JS jetstream.JetStream } + +// Follow is DeadLetterFollower's. +func (j JetStreamDeadLetters) Follow(ctx context.Context, stream string, since time.Time, heard func(GivenUp)) (func(), error) { + cons, err := j.JS.OrderedConsumer(ctx, DeadLettersStream, jetstream.OrderedConsumerConfig{ + FilterSubjects: []string{DeadLetterFilter(stream)}, DeliverPolicy: jetstream.DeliverByStartTimePolicy, OptStartTime: &since}) + if err != nil { + return nil, err + } + cc, err := cons.Consume(func(m jetstream.Msg) { + if g, ok := GivenUpOf(m.Headers()); ok { + heard(g) + } + }) + if err != nil { + return nil, err + } + return cc.Stop, nil +} + +// DeadLetterHearing follows the build seats' given-up asks into Dead. Make it with NewDeadLetterHearing. +type DeadLetterHearing struct { + follower DeadLetterFollower + streams []string + dead *DeadLetters + logf func(string, ...any) + said func(Hearing) + ctx context.Context + since time.Time + + mu sync.Mutex + stops map[string]func() + why map[string]string + trying bool + again bool + told *Hearing + wg sync.WaitGroup +} + +// NewDeadLetterHearing starts hearing the given-up asks of each language's build seat stream, in the background. said +// is told each change of whether it hears (nil tells nothing); ctx ends it. +func NewDeadLetterHearing(ctx context.Context, follower DeadLetterFollower, languages []string, dead *DeadLetters, + logf func(string, ...any), said func(Hearing)) *DeadLetterHearing { + h := &DeadLetterHearing{follower: follower, dead: dead, logf: logf, said: said, ctx: ctx, since: time.Now(), + stops: map[string]func(){}, why: map[string]string{}} + for _, l := range languages { + s := SeatStream(l) + h.streams = append(h.streams, s) + h.why[s] = "not tried yet" + } + if h.logf == nil { + h.logf = func(string, ...any) {} + } + h.Again() + return h +} + +// Deaf says why a stream is not heard, or "" when every one is. +func (h *DeadLetterHearing) Deaf() string { + h.mu.Lock() + defer h.mu.Unlock() + return h.deafLocked() +} + +func (h *DeadLetterHearing) deafLocked() string { + var out []string + for _, s := range h.streams { + if why, deaf := h.why[s]; deaf { + out = append(out, s+": "+why) + } + } + if len(out) == 0 { + return "" + } + return DeadLettersStream + " is not heard, so an ask a build seat gives up on is waited for until its bound: " + + strings.Join(out, "; ") +} + +// Again tries, in the background, every stream not heard yet; a call while it tries makes it try once more after. +func (h *DeadLetterHearing) Again() { + h.mu.Lock() + if h.trying { + h.again = true + h.mu.Unlock() + return + } + h.trying = true + h.mu.Unlock() + h.wg.Add(1) + go func() { + defer h.wg.Done() + for { + h.try() + h.mu.Lock() + if !h.again || h.ctx.Err() != nil { + h.trying = false + h.mu.Unlock() + return + } + h.again = false + h.mu.Unlock() + } + }() +} + +// Wait returns when no try runs. +func (h *DeadLetterHearing) Wait() { h.wg.Wait() } + +// Stop ends every stream's following. +func (h *DeadLetterHearing) Stop() { + h.mu.Lock() + defer h.mu.Unlock() + for s, stop := range h.stops { + stop() + delete(h.stops, s) + } +} + +func (h *DeadLetterHearing) try() { + for _, s := range h.streams { + h.mu.Lock() + _, deaf := h.why[s] + h.mu.Unlock() + if !deaf || h.ctx.Err() != nil { + continue + } + stream := s + stop, err := h.follower.Follow(h.ctx, stream, h.since, func(g GivenUp) { + h.logf("the ask %s on %s was given up after %d deliveries at %s: the wait for its step ends", g.AskID, g.Stream, + g.Deliveries, g.At.Format(time.RFC3339)) + h.dead.Heard(g) + }) + h.mu.Lock() + if err != nil { + h.why[stream] = err.Error() + h.logf("the asks %s gives up on are not heard (%s of %s); tried again at the next reconnect or ask: %v", + stream, DeadLetterFilter(stream), DeadLettersStream, err) + } else { + delete(h.why, stream) + h.stops[stream] = stop + h.logf("the asks %s gives up on are heard from %s", stream, DeadLettersStream) + } + h.mu.Unlock() + } + h.tell() +} + +// tell says the hearing when it changed. +func (h *DeadLetterHearing) tell() { + h.mu.Lock() + why := h.deafLocked() + now := Hearing{Heard: why == "", Streams: append([]string(nil), h.streams...), Why: why, At: time.Now().UTC()} + same := h.told != nil && h.told.Heard == now.Heard && h.told.Why == now.Why + if !same { + h.told = &now + } + h.mu.Unlock() + if !same && h.said != nil { + h.said(now) + } +} + +// HearingState is the name of the state an asker keeps its Hearing in (`_hearing`, key HearingKey), which its +// health tool reads. +const HearingState = "hearing" + +// HearingKeeper is where a Hearing is kept: a jetstream.KeyValue, or a process.AwaitedBucket for a state that may not be +// on the bus at the part's start (issue 1790), which keeps the latest until it appears and answers process.ErrNotYet. +type HearingKeeper interface { + Put(ctx context.Context, key string, value []byte) (uint64, error) +} + +// SayHearing is a DeadLetterHearing's said that keeps each Hearing in kv under HearingKey; nil kv keeps nothing and +// says so once. A write kv keeps for later (process.ErrNotYet) is not an error: the awaited bucket said it waits. +func SayHearing(ctx context.Context, kv HearingKeeper, logf func(string, ...any)) func(Hearing) { + if kv == nil { + if logf != nil { + logf("whether %s is heard is not kept: the state %s is not there, so the health tool cannot say it", DeadLettersStream, HearingState) + } + return nil + } + return func(h Hearing) { + data, err := json.Marshal(h) + if err == nil { + _, err = kv.Put(ctx, HearingKey, data) + } + if err != nil && !errors.Is(err, process.ErrNotYet) && logf != nil { + logf("whether %s is heard could not be kept: %v", DeadLettersStream, err) + } + } +} + +// HearingHealth is the health answer of an asker's kept Hearing (raw, empty when none is kept): healthy while it hears +// DEAD_LETTERS, unhealthy saying why while it does not or has not said. +func HearingHealth(raw []byte) health.Answer { + if len(raw) == 0 { + return health.Unhealthy("it has not said whether it hears " + DeadLettersStream + " (issue 734): it is starting, or its state " + HearingState + " is not there") + } + var h Hearing + if err := json.Unmarshal(raw, &h); err != nil { + return health.Unhealthy("what it keeps of hearing " + DeadLettersStream + " is not a hearing: " + err.Error()) + } + if !h.Heard { + return health.Unhealthy(h.Why + " (since " + h.At.Format(time.RFC3339) + "; tried again at its next reconnect or ask)") + } + return health.Healthy("it hears " + DeadLettersStream + " for " + strings.Join(h.Streams, ", ")) +} diff --git a/modules/mesh-delivery-builder/buildrun/modules.go b/modules/mesh-delivery-builder/buildrun/modules.go new file mode 100644 index 0000000..db15aed --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/modules.go @@ -0,0 +1,7 @@ +package buildrun + +// The build modules the mesh runs, by the language each builds (ADR 0469). +const ( + GoBuildModule = "mesh-build-go" + NodeBuildModule = "mesh-build-node" +) diff --git a/modules/mesh-delivery-builder/buildrun/places.go b/modules/mesh-delivery-builder/buildrun/places.go new file mode 100644 index 0000000..d96dda6 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/places.go @@ -0,0 +1,145 @@ +package buildrun + +import ( + "context" + "fmt" + "sort" + "sync" +) + +// DefaultCapacity is the build capacity of a node that has no setting: one build step at a time. +const DefaultCapacity = 1 + +// Places counts the build steps a part runs on each node and holds a lane back while its node is at its +// capacity (the operator's decision of 2026-10-21; ADR 0478 clause 3). A lane is the fetch and the builds of one build +// module on one node, run one step at a time, so a lane holds one place while it runs. A node is chosen for a lane by +// reserving a place on it at the moment of the choice (ChooseNodes), so that builds choosing at once see each other's +// choices and none waits on a full node while another has room. A lane waits for its place by watching the ends of the +// other lanes, not by a timer: ending a lane wakes the waiting ones. +type Places struct { + // Capacity is the number of steps at once by node; a node not in it has DefaultCapacity. + Capacity map[string]int + + mu sync.Mutex + running map[string]int // lanes holding a place + reserved map[string]int // lanes chosen for the node that do not hold a place yet + changed chan struct{} // closed, and replaced, each time a lane ends +} + +// CapacityOf is how many steps the part runs on the node at once: DefaultCapacity for a node the setting does not +// name, and 0 for one it names so, where the part never builds. +func (p *Places) CapacityOf(node string) int { + if n, ok := p.Capacity[node]; ok && n >= 0 { + return n + } + return DefaultCapacity +} + +// Running is the steps in flight now, by node. +func (p *Places) Running() map[string]int { + p.mu.Lock() + defer p.mu.Unlock() + out := make(map[string]int, len(p.running)) + for n, c := range p.running { + if c > 0 { + out[n] = c + } + } + return out +} + +// Reserved is the lanes chosen for a node that hold no place yet, by node. +func (p *Places) Reserved() map[string]int { + p.mu.Lock() + defer p.mu.Unlock() + out := make(map[string]int, len(p.reserved)) + for n, c := range p.reserved { + if c > 0 { + out[n] = c + } + } + return out +} + +func (p *Places) init() { + if p.running == nil { + p.running = map[string]int{} + } + if p.reserved == nil { + p.reserved = map[string]int{} + } + if p.changed == nil { + p.changed = make(chan struct{}) + } +} + +// Acquire takes a place on the node, waiting while the node is at its capacity, and answers the function that gives it +// back. A place reserved for the node by a choice is the one taken. It returns an error when ctx ends first, and then +// gives up the reservation too. +func (p *Places) Acquire(ctx context.Context, node string) (release func(), err error) { + for { + p.mu.Lock() + p.init() + if p.running[node] < p.CapacityOf(node) { + p.running[node]++ + if p.reserved[node] > 0 { + p.reserved[node]-- + } + p.mu.Unlock() + var once sync.Once + return func() { + once.Do(func() { + p.mu.Lock() + p.running[node]-- + close(p.changed) + p.changed = make(chan struct{}) + p.mu.Unlock() + }) + }, nil + } + wait := p.changed + p.mu.Unlock() + select { + case <-wait: + case <-ctx.Done(): + p.Cancel(node) + return nil, fmt.Errorf("waiting for a place on %s: %w", node, ctx.Err()) + } + } +} + +// Cancel gives up a place reserved for the node that will not be taken. +func (p *Places) Cancel(node string) { + p.mu.Lock() + defer p.mu.Unlock() + p.init() + if p.reserved[node] > 0 { + p.reserved[node]-- + } +} + +// choose orders the nodes (those with room first, counting the lanes running and the lanes reserved, then the fewest of +// them, then by name), keeps the first want and reserves a place on each, all at once. +func (p *Places) choose(nodes []string, want int) []string { + p.mu.Lock() + defer p.mu.Unlock() + p.init() + load := func(n string) int { return p.running[n] + p.reserved[n] } + sort.SliceStable(nodes, func(i, j int) bool { + ri, rj := load(nodes[i]) < p.CapacityOf(nodes[i]), load(nodes[j]) < p.CapacityOf(nodes[j]) + if ri != rj { + return ri + } + if load(nodes[i]) != load(nodes[j]) { + return load(nodes[i]) < load(nodes[j]) + } + return nodes[i] < nodes[j] + }) + if len(nodes) > want { + nodes = nodes[:want] + } + for _, n := range nodes { + p.reserved[n]++ + } + return nodes +} diff --git a/modules/mesh-delivery-builder/buildrun/places_test.go b/modules/mesh-delivery-builder/buildrun/places_test.go new file mode 100644 index 0000000..04386e7 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/places_test.go @@ -0,0 +1,104 @@ +package buildrun + +import ( + "context" + "strings" + "testing" + "time" +) + +func acquireAsync(p *Places, node string) chan func() { + got := make(chan func(), 1) + go func() { + release, err := p.Acquire(context.Background(), node) + if err != nil { + close(got) + return + } + got <- release + }() + return got +} + +func TestThreeStepsOnANodeOfCapacityTwoRunTwoAtOnceAndTheThirdStartsWhenOneEnds(t *testing.T) { // round 3 + p := &Places{Capacity: map[string]int{"ace": 2}} + a, b := acquireAsync(p, "ace"), acquireAsync(p, "ace") + ra, rb := <-a, <-b + third := acquireAsync(p, "ace") + select { + case <-third: + t.Fatal("a third step started on a node of capacity 2 with two running") + case <-time.After(10 * time.Millisecond): + } + if p.Running()["ace"] != 2 { + t.Errorf("running = %v, want 2 on ace", p.Running()) + } + ra() + select { + case r := <-third: + r() + case <-time.After(2 * time.Second): + t.Fatal("the third step did not start when one ended") + } + rb() +} + +func TestANodeWithNoSettingRunsOneStepAtATime(t *testing.T) { // round 3 + p := &Places{Capacity: map[string]int{"shanks": 3}} + first := <-acquireAsync(p, "g14") + second := acquireAsync(p, "g14") + select { + case <-second: + t.Fatal("two steps on a node without a setting") + case <-time.After(10 * time.Millisecond): + } + first() + r := <-second + r() + if p.CapacityOf("g14") != 1 || p.CapacityOf("shanks") != 3 { + t.Errorf("capacities = %d, %d", p.CapacityOf("g14"), p.CapacityOf("shanks")) + } +} + +func TestAWaitingStepStopsWhenItsContextEnds(t *testing.T) { // round 3 + p := &Places{} + r := <-acquireAsync(p, "ace") + defer r() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { _, err := p.Acquire(ctx, "ace"); done <- err }() + cancel() + select { + case err := <-done: + if err == nil { + t.Error("Acquire returned no error for a cancelled wait") + } + case <-time.After(2 * time.Second): + t.Fatal("a cancelled wait did not return") + } +} + +// The operator's decision of 2026-10-21 13:40: the builder builds on shanks and ace only; a capacity of 0 keeps it off a node. +func TestACapacityOfZeroKeepsThePartOffTheNodeAndANodeNotNamedKeepsOne(t *testing.T) { // issue 1661 + p := &Places{Capacity: map[string]int{"shanks": 3, "ace": 2, "g14": 0, "novox": 0}} + if p.CapacityOf("g14") != 0 || p.CapacityOf("novox") != 0 || p.CapacityOf("ace") != 2 || p.CapacityOf("other") != DefaultCapacity { + t.Errorf("capacities = %d %d %d %d, want 0 0 2 and the default for a node not named", p.CapacityOf("g14"), p.CapacityOf("novox"), p.CapacityOf("ace"), p.CapacityOf("other")) + } + entries := tcs("go", "go1.27.1", "ace", "g14", "novox", "shanks", "other") + for i := 0; i < 6; i++ { + nodes, _, err := ChooseNodes("go", entries, p, nil, 2) + if err != nil { + t.Fatal(err) + } + for _, n := range nodes { + if n == "g14" || n == "novox" { + t.Fatalf("choice %d: %v includes a node of capacity 0", i, nodes) + } + } + } + // Only nodes of capacity 0 report: nothing qualifies, and the build says why. + _, _, err := ChooseNodes("go", tcs("go", "go1.27.1", "g14", "novox"), p, nil, 2) + if err == nil || !strings.Contains(err.Error(), "capacity") { + t.Errorf("only capacity-0 nodes: error = %v, want it to name the capacity", err) + } +} diff --git a/modules/mesh-delivery-builder/buildrun/presence.go b/modules/mesh-delivery-builder/buildrun/presence.go new file mode 100644 index 0000000..63f8243 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/presence.go @@ -0,0 +1,53 @@ +package buildrun + +import ( + "sync" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// PresenceSet holds which nodes are online, as the watchman writes it (mesh-delivery-watchman_presence, key ``, the +// node-engine's connection): a node asleep or off the bus is offline and its build module cannot answer. A node the +// watchman has no key of is not online. Safe for use from the watch and from a build at once. +type PresenceSet struct { + mu sync.Mutex + online map[string]bool +} + +// Put takes a change of the presence bucket in and answers whether a node's online state changed. A part's key and a +// value that is not a presence are left out. +func (s *PresenceSet) Put(key string, value []byte, deleted bool) bool { + node, part, _, err := delivery.ParsePresenceKey(key) + if err != nil || part != "" { + return false + } + on := false + if !deleted { + p, err := delivery.DecodePresence(value) + if err != nil { + return false + } + on = p.Online + } + s.mu.Lock() + defer s.mu.Unlock() + if s.online[node] == on { + return false + } + if s.online == nil { + s.online = map[string]bool{} + } + if on { + s.online[node] = true + } else { + delete(s.online, node) + } + return true +} + +// Online is whether the watchman last saw the node online. +func (s *PresenceSet) Online(node string) bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.online[node] +} diff --git a/modules/mesh-delivery-builder/buildrun/presence_test.go b/modules/mesh-delivery-builder/buildrun/presence_test.go new file mode 100644 index 0000000..5157609 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/presence_test.go @@ -0,0 +1,43 @@ +package buildrun + +import ( + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func presenceValue(t *testing.T, online bool) []byte { + t.Helper() + b, err := delivery.EncodePresence(delivery.Presence{Online: online}) + if err != nil { + t.Fatal(err) + } + return b +} + +func TestPresenceSetKnowsOnlyNodesTheWatchmanSawOnline(t *testing.T) { // 1607 review 3 + var s PresenceSet + if s.Online("ace") { + t.Error("a node the watchman never wrote is online") + } + if !s.Put("ace", presenceValue(t, true), false) || !s.Online("ace") { + t.Error("ace online was not taken in") + } + if s.Put("ace", presenceValue(t, true), false) { + t.Error("the same presence again counted as a change") + } + if !s.Put("ace", presenceValue(t, false), false) || s.Online("ace") { + t.Error("ace going offline was not taken in") + } + s.Put("g14", presenceValue(t, true), false) + if !s.Put("g14", nil, true) || s.Online("g14") { + t.Error("a deleted key is still online") + } + // A part's instance (part..) is not a node. + if s.Put("part.mesh-delivery-builder.novox", presenceValue(t, true), false) || s.Online("part") { + t.Error("a part's key was taken for a node") + } + if s.Put("novox", []byte("junk"), false) { + t.Error("junk counted") + } +} diff --git a/modules/mesh-delivery-builder/buildrun/standing.go b/modules/mesh-delivery-builder/buildrun/standing.go new file mode 100644 index 0000000..c0b92e9 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/standing.go @@ -0,0 +1,225 @@ +package buildrun + +import ( + "context" + "errors" + "fmt" + "maps" + "slices" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// A build module writes . of its toolchains state on every node it runs on, and nothing deletes the key +// when the module leaves a node: no part writes another module's state, and the module is gone. A report of a node the +// build module no longer runs on is therefore no builder's (issue 1940: mesh-build-node taken off ace on 2026-10-23 left +// ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree"). +// +// ToolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a +// ToolchainSet): an entry is passed on only while the placement (the seat mesh-placement's state `placement`, held by +// the assigner, delivery.PlacementBucket, read by delivery.Placed; ADR 0567, ADR 0549 §3) places its build module on +// its node, and only once it was written since that placement; it is passed on as deleted otherwise, said once, and +// passed on when its node writes it again. A report written before the placement is one an earlier run of the module +// left (issue 1940, 2026-10-25 20:56: mesh-build-node put on ace again brought ace.typescript v26.12.0 of 2026-10-23 +// back into the agreement before the module ran there). Until the placement is settled, and while no node's placement +// is held (before the assigner's adopt writes the first one, or where the state cannot be read), every entry is passed +// on. Builders that run and report different versions still disagree: that refusal is the guard of novox/hq ADR 0465, +// and it stays. +// +// This replaces the copies of the gate the indexer, the builder and the checker each kept over the controller's +// assignments (issue 1940's subtask, issue 774: those go with the controller's lease, ADR 0549 §6). +type ToolchainGate struct { + mu sync.Mutex + modules map[string]bool // the build modules whose toolchains pass the gate + forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds + logf func(format string, a ...any) + placed *delivery.Placed + known bool // the placement is settled + raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it + left map[string]string // / -> why it was said left out, so each reason is said once +} + +// NewToolchainGate is a gate over the toolchains of modules, passing what stands on to forward and saying with logf what +// it leaves out. +func NewToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *ToolchainGate { + g := &ToolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, placed: delivery.NewPlaced(), + raw: map[string]map[string][]byte{}, left: map[string]string{}} + for _, m := range modules { + g.modules[m] = true + } + return g +} + +// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed. +func (g *ToolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool { + g.mu.Lock() + defer g.mu.Unlock() + if g.raw[module] == nil { + g.raw[module] = map[string][]byte{} + } + if deleted { + delete(g.raw[module], key) + } else { + g.raw[module][key] = value + } + return g.pass(module, key) +} + +// Placement takes one revision of the seat mesh-placement's placement in (at is when the bus wrote it) and answers whether what it +// passed on changed. A value that does not read is said, and places nothing on its node. +func (g *ToolchainGate) Placement(key string, value []byte, deleted bool, at time.Time) bool { + g.mu.Lock() + defer g.mu.Unlock() + was := g.placed.Seeded() + node, changed, err := g.placed.Put(key, value, deleted, at) + if err != nil { + g.logf("%s: %v (issue 1940)", delivery.PlacementBucket, err) + } + if !g.known { + return false + } + if was != g.placed.Seeded() { + // The first placement came, or the last went: every report's standing changed. + return g.all() + } + return g.again(node, changed) +} + +// Settled says the placement's history is read: from now on only the reports of placed nodes pass, once a node's +// placement is held. It answers whether what it passed on changed. +func (g *ToolchainGate) Settled() bool { + g.mu.Lock() + defer g.mu.Unlock() + g.known = true + return g.all() +} + +// all passes on every report again. The caller holds mu. +func (g *ToolchainGate) all() bool { + changed := false + for _, module := range slices.Sorted(maps.Keys(g.raw)) { + for _, k := range slices.Sorted(maps.Keys(g.raw[module])) { + changed = g.pass(module, k) || changed + } + } + return changed +} + +// again passes on every report of node by a build module among modules. The caller holds mu. +func (g *ToolchainGate) again(node string, modules []string) bool { + changed := false + for _, module := range modules { + if !g.modules[module] { + continue + } + for _, k := range slices.Sorted(maps.Keys(g.raw[module])) { + if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node { + changed = g.pass(module, k) || changed + } + } + } + return changed +} + +// pass hands one key on as it stands: its value when its node is placed and wrote it since (or the placement is not +// known), deleted otherwise. The caller holds mu. +func (g *ToolchainGate) pass(module, key string) bool { + value, held := g.raw[module][key] + if !held { + return g.forward(module, key, nil, true) + } + node, _, err := delivery.ParseToolchainKey(key) + why := "" + if err == nil && g.known && g.placed.Seeded() { + why = g.leftOut(module, node, value) + } + if why == "" { + delete(g.left, module+"/"+key) + return g.forward(module, key, value, false) + } + if g.left[module+"/"+key] != why { + g.left[module+"/"+key] = why + g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why) + } + return g.forward(module, key, nil, true) +} + +// leftOut is why a node's report is no builder's, or empty when it counts: the catalogue does not place the module on its +// node, or the report was written before it did. A value that is no toolchain is passed on for its reader to refuse. +// The caller holds mu. +func (g *ToolchainGate) leftOut(module, node string, value []byte) string { + since, placed := g.placed.Since(node, module) + if !placed { + return fmt.Sprintf("the catalogue does not place %s on %s, so it is no builder", module, node) + } + tc, err := delivery.DecodeToolchain(value) + if err == nil && tc.At.Before(since) { + return fmt.Sprintf("it was written at %s, before %s was placed on %s at %s: it counts once %s writes it again", + tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node) + } + return "" +} + +// FollowPlacement reads the seat mesh-placement's placement with its history into gate, says it settled, and follows it; changed +// is told whenever what the gate passes on changed. Where the state cannot be read or watched (not there, or this part +// may not read it) it says so and the gate passes every report on: not fatal, since an indexer, builder or checker that +// stops for it stops every delivery (issue 1945). An error of the watch after that ends on the channel. +func FollowPlacement(ctx context.Context, js process.BucketOpener, gate *ToolchainGate, changed func(context.Context), logf func(string, ...any)) (<-chan error, error) { + errs := make(chan error, 1) + kv, err := js.KeyValue(ctx, delivery.PlacementBucket) + if err != nil { + logf("the state %s cannot be read, so every node's toolchain report counts, a build node's the catalogue does not place too (issue 1940): %v", delivery.PlacementBucket, err) + return errs, nil + } + w, err := kv.WatchAll(ctx, jetstream.IncludeHistory()) + if err != nil { + logf("the state %s cannot be watched, so every node's toolchain report counts, a build node's the catalogue does not place too (issue 1940): %v", delivery.PlacementBucket, err) + return errs, nil + } + put := func(e jetstream.KeyValueEntry) bool { + return gate.Placement(e.Key(), e.Value(), e.Operation() != jetstream.KeyValuePut, e.Created()) + } + for settled := false; !settled; { + select { + case <-ctx.Done(): + _ = w.Stop() + return errs, ctx.Err() + case e, ok := <-w.Updates(): + if !ok { + return errs, errors.New("the watch of " + delivery.PlacementBucket + " ended before it settled") + } + if e == nil { + settled = true + continue + } + put(e) + } + } + if gate.Settled() { + changed(ctx) + } + go func() { + defer func() { _ = w.Stop() }() + for { + select { + case <-ctx.Done(): + errs <- ctx.Err() + return + case e, ok := <-w.Updates(): + if !ok { + errs <- errors.New("the watch of " + delivery.PlacementBucket + " ended") + return + } + if e != nil && put(e) { + changed(ctx) + } + } + } + }() + return errs, nil +} diff --git a/modules/mesh-delivery-builder/buildrun/standing_test.go b/modules/mesh-delivery-builder/buildrun/standing_test.go new file mode 100644 index 0000000..6a55ef4 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/standing_test.go @@ -0,0 +1,263 @@ +package buildrun + +import ( + "bytes" + "context" + "errors" + "fmt" + "log" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func gateReport(t *testing.T, node, version string, at time.Time) (string, []byte) { + t.Helper() + key, err := delivery.ToolchainKey(node, "typescript") + if err != nil { + t.Fatal(err) + } + value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at}) + if err != nil { + t.Fatal(err) + } + return key, value +} + +// placing is a value of the state `placement` placing modules on node. +func placing(t *testing.T, node string, modules ...string) (string, []byte) { + t.Helper() + p := delivery.Placement{Node: node} + for _, m := range modules { + p.Modules = append(p.Modules, delivery.PlacedModule{Module: m}) + } + values, err := delivery.PlacementValues(p) + if err != nil { + t.Fatal(err) + } + key, _ := delivery.PlacementNodeKey(node) + return key, values[key] +} + +// placingLegacy is placing in the form before, `placement.` (research 072 row 37). +func placingLegacy(t *testing.T, node string, modules ...string) (string, []byte) { + t.Helper() + p := delivery.Placement{Node: node} + for _, m := range modules { + p.Modules = append(p.Modules, delivery.PlacedModule{Module: m}) + } + key, _ := delivery.LegacyPlacementKey(node) + value, err := delivery.EncodePlacement(p) + if err != nil { + t.Fatal(err) + } + return key, value +} + +func agreedOf(set *ToolchainSet) (string, string) { + tc, none := delivery.AgreedToolchain(set.Entries("typescript")) + return tc.Version, none +} + +// Issue 1940, the gate over the catalogue's placement (issue 774, ADR 0549 §3): a build node's report counts only while +// the placement places its build module on its node, and only once written since; before the placement is settled, and +// while no node's placement is held, every report counts. The live cases: mesh-build-node taken off ace left +// ace.typescript at v26.12.0 against v26.12.1 on the three others; put on ace again, its report of 2026-10-23 06:29 came +// back before the module ran there. +func TestToolchainGate(t *testing.T) { + old := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run + placed := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node placed on ace again + three := []string{"g14", "novox", "shanks"} + start := func(t *testing.T, seeded bool) (*ToolchainGate, *ToolchainSet, *[]string) { + set := &ToolchainSet{} + said := &[]string{} + gate := NewToolchainGate([]string{GoBuildModule, NodeBuildModule}, set.Put, func(f string, a ...any) { *said = append(*said, fmt.Sprintf(f, a...)) }) + for _, node := range three { + if seeded { + key, value := placing(t, node, NodeBuildModule) + gate.Placement(key, value, false, old.Add(-time.Hour)) + } + key, value := gateReport(t, node, "v26.12.1", placed.Add(time.Minute)) + gate.Toolchain(NodeBuildModule, key, value, false) + } + return gate, set, said + } + + t.Run("not placed on ace: its report is no builder's", func(t *testing.T) { + gate, set, said := start(t, true) + key, value := gateReport(t, "ace", "v26.12.0", old) + gate.Toolchain(NodeBuildModule, key, value, false) + if a := delivery.ToolchainAgreement(set.Entries("typescript")); len(a.SittingOut) != 1 || a.SittingOut[0].Node != "ace" { // ADR 0584: the majority is agreed, ace's report counts as the one that sits out + t.Fatalf("before the placement is settled every report counts; got %+v", a) + } + k, v := placing(t, "ace", GoBuildModule) // another build module on ace: no matter + gate.Placement(k, v, false, old) + if !gate.Settled() { + t.Fatal("settling left ace's report out, so what was passed on changed") + } + if v, none := agreedOf(set); v != "v26.12.1" || none != "" { + t.Fatalf("the placed nodes agree on v26.12.1; got %q, %q", v, none) + } + if len(*said) != 1 || !strings.Contains((*said)[0], "does not place mesh-build-node on ace") { + t.Fatalf("ace's report is said left out once; said %q", *said) + } + if gate.Toolchain(NodeBuildModule, key, value, false) { + t.Fatal("a report of a node not placed changed what was passed on") + } + // Placed on ace with a report written since: it counts, and the builders really disagree. + k, v = placing(t, "ace", GoBuildModule, NodeBuildModule) + gate.Placement(k, v, false, old.Add(-time.Minute)) + if a := delivery.ToolchainAgreement(set.Entries("typescript")); a.Toolchain.Version != "v26.12.1" || len(a.SittingOut) != 1 || a.SittingOut[0].Version != "v26.12.0" { + t.Fatalf("builders that run different versions: the majority is agreed and ace sits out; got %+v", a) + } + // Taken off ace again (its key deleted): its report goes again. + if !gate.Placement(k, nil, true, placed) { + t.Fatal("taking ace's placement away passed nothing on") + } + if v, none := agreedOf(set); v != "v26.12.1" || none != "" { + t.Fatalf("got %q, %q", v, none) + } + // A report deleted from the bucket goes whether its node is placed or not. + key, _ = gateReport(t, "g14", "v26.12.1", placed) + if !gate.Toolchain(NodeBuildModule, key, nil, true) || len(set.Entries("typescript")) != 2 { + t.Fatalf("deleting g14's report: %d entries left, want novox and shanks", len(set.Entries("typescript"))) + } + }) + + for _, c := range []struct { + name string + at time.Time // when ace wrote its report + counts bool + }{ + {"a report left before the placement", old, false}, + {"a report written since the placement", placed.Add(30 * time.Second), true}, + } { + t.Run(c.name, func(t *testing.T) { + gate, set, said := start(t, true) + k, v := placing(t, "ace") // ace's key, first heard placing nothing + gate.Placement(k, v, false, old.Add(-time.Hour)) + gate.Settled() + k, v = placing(t, "ace", NodeBuildModule) + if gate.Placement(k, v, false, placed) { + t.Fatal("a placement with no report passed something on") + } + key, value := gateReport(t, "ace", "v26.12.0", c.at) + if got := gate.Toolchain(NodeBuildModule, key, value, false); got != c.counts { + t.Fatalf("ace's report passed on: %v, want %v", got, c.counts) + } + // ace's placement written again with another module beside: its standing does not move, nothing is said again. + k, v = placing(t, "ace", GoBuildModule, NodeBuildModule) + if gate.Placement(k, v, false, placed.Add(time.Minute)) { + t.Fatal("the same placement of mesh-build-node changed what was passed on") + } + if c.counts { + return + } + if v, none := agreedOf(set); v != "v26.12.1" || none != "" { + t.Fatalf("ace's left report is no builder's; got %q, %q", v, none) + } + if len(*said) != 1 || !strings.Contains((*said)[0], "before mesh-build-node was placed on ace at 2026-10-25T19:56:45Z") { + t.Fatalf("said %q, want once why ace's report is left out", *said) + } + key, value = gateReport(t, "ace", "v26.12.1", placed.Add(time.Minute)) + if !gate.Toolchain(NodeBuildModule, key, value, false) || len(set.Entries("typescript")) != 4 { + t.Fatal("ace's new report was not passed on") + } + }) + } + + t.Run("no placement held: every report counts", func(t *testing.T) { + gate, set, _ := start(t, false) + key, value := gateReport(t, "ace", "v26.12.0", old) + gate.Toolchain(NodeBuildModule, key, value, false) + gate.Settled() + if n := len(set.Entries("typescript")); n != 4 { + t.Fatalf("before the first placement file every report counts; got %d", n) + } + k, v := placing(t, "g14", NodeBuildModule) + if !gate.Placement(k, v, false, old.Add(-time.Hour)) || len(set.Entries("typescript")) != 1 { + t.Fatalf("the first placement leaves every node it does not place out; got %d", len(set.Entries("typescript"))) + } + if !gate.Placement(k, nil, true, placed) || len(set.Entries("typescript")) != 4 { + t.Fatal("the last placement gone: every report counts again") + } + }) + + t.Run("the history: a module in the first revision heard of a key stands since before it", func(t *testing.T) { + p := delivery.NewPlaced() + // ace in the form before, then converted: its module stands since before, the legacy key read no more. + k, v := placingLegacy(t, "ace", NodeBuildModule) + p.Put(k, v, false, old) + k, v = placing(t, "ace", NodeBuildModule) + if _, changed, err := p.Put(k, v, false, placed); len(changed) != 0 || err != nil { + t.Errorf("the conversion of ace's key changed %v, %v", changed, err) + } + if _, changed, _ := p.Put("ace."+NodeBuildModule, []byte("{}"), false, placed); len(changed) != 0 { + t.Errorf("a module key says which modules stand: %v", changed) + } + if k, v := placingLegacy(t, "ace"); true { + if _, changed, _ := p.Put(k, v, false, placed); len(changed) != 0 { + t.Errorf("the legacy key is read after the node key: %v", changed) + } + } + k2, v2 := placing(t, "g14") + p.Put(k2, v2, false, old) + k2, v2 = placing(t, "g14", NodeBuildModule) + if _, changed, _ := p.Put(k2, v2, false, placed); strings.Join(changed, ",") != NodeBuildModule { + t.Errorf("a module placed says changed %v", changed) + } + if _, _, err := p.Put("placement.novox", []byte("not json"), false, old); err == nil { + t.Error("a value that does not read was not said") + } + if s, ok := p.Since("ace", NodeBuildModule); !ok || !s.IsZero() { + t.Errorf("ace's module stands since %v, %v; want the zero time", s, ok) + } + if s, ok := p.Since("g14", NodeBuildModule); !ok || !s.Equal(placed) { + t.Errorf("g14's module stands since %v, %v; want the revision that placed it", s, ok) + } + if got := p.PlacedNodes(NodeBuildModule); strings.Join(got, ",") != "ace,g14" || len(p.ByNode()) != 2 || !p.Seeded() { + t.Errorf("PlacedNodes = %v, ByNode = %v", got, p.ByNode()) + } + }) + + t.Run("a placement the bus refuses is said, and the part goes on", func(t *testing.T) { + var logs bytes.Buffer + logger := log.New(&logs, "", 0) + refused := errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_placement") + gate, _, _ := start(t, false) + for _, bus := range []refusingBus{{err: refused}, {refuseOpen: true, err: refused}} { + if _, err := FollowPlacement(context.Background(), bus, gate, func(context.Context) {}, logger.Printf); err != nil { + t.Errorf("a refused placement stopped the start: %v", err) + } + } + if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") { + t.Errorf("the refusals were not said: %q", logs.String()) + } + }) +} + +// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err. +type refusingBus struct { + refuseOpen bool + err error +} + +func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { + if r.refuseOpen { + return nil, r.err + } + return refusedWatch{err: r.err}, nil +} + +type refusedWatch struct { + jetstream.KeyValue + err error +} + +func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + return nil, r.err +} diff --git a/modules/mesh-delivery-builder/buildrun/steps.go b/modules/mesh-delivery-builder/buildrun/steps.go new file mode 100644 index 0000000..f08e002 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/steps.go @@ -0,0 +1,67 @@ +package buildrun + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Caller calls a tool of a module on one named machine (mesh-sdk process.Conn.CallOn; ADR 0469 clause 3). +type Caller interface { + CallOn(node, key string, body any) (json.RawMessage, error) +} + +// Steps waits for the end of a build module's long step: the value of its key `.` in the module's steps state +// (ADR 0469 clause 2) once its state is not running. It is a watch of that key, not a poll; it returns when the step +// ends or ctx does. +type Steps interface { + Wait(ctx context.Context, module, key string) (delivery.Step, error) +} + +// Runner runs a build module's long steps: the call answers at once with the step's key, and the result is read from that +// key when the step ends (ADR 0469 clause 2). +type Runner struct { + Caller Caller + Steps Steps + // Bound is how long a step may run before the builder stops waiting for it. + Bound time.Duration +} + +// Run calls tool of the build module on node with args, waits for the step's end and answers its result. A step that +// failed or was refused is an error naming the node, the state and the reason. +func (r Runner) Run(ctx context.Context, module, node, tool string, args map[string]any) (json.RawMessage, error) { + raw, err := r.Caller.CallOn(node, module+"."+tool, args) + if err != nil { + return nil, fmt.Errorf("%s on %s: %w", tool, node, err) + } + var answer struct { + Step string `json:"step"` + Key string `json:"key"` + State string `json:"state"` + } + if err := json.Unmarshal(raw, &answer); err != nil { + return nil, fmt.Errorf("%s on %s answered what is not a step: %w", tool, node, err) + } + if answer.Key == "" { + return nil, fmt.Errorf("%s on %s answered no step key: %s", tool, node, raw) + } + if n, _, err := delivery.ParseStepKey(answer.Key); err != nil || n != node { + return nil, fmt.Errorf("%s on %s answered the step key %q, which is not a key of that node", tool, node, answer.Key) + } + wait, cancel := context.WithTimeoutCause(ctx, r.Bound, fmt.Errorf("the step %s did not end within %s", answer.Key, r.Bound)) + defer cancel() + step, err := r.Steps.Wait(wait, module, answer.Key) + if err != nil { + if cause := context.Cause(wait); cause != nil && ctx.Err() == nil { + err = cause + } + return nil, fmt.Errorf("%s on %s: %w", tool, node, err) + } + if step.State != delivery.StepPassed { + return nil, fmt.Errorf("%s on %s: the step %s %s: %s", tool, node, answer.Key, step.State, step.Reason) + } + return step.Result, nil +} diff --git a/modules/mesh-delivery-builder/buildrun/steps_test.go b/modules/mesh-delivery-builder/buildrun/steps_test.go new file mode 100644 index 0000000..9d6d024 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/steps_test.go @@ -0,0 +1,116 @@ +package buildrun + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +type call struct { + node, key string + body any +} + +// fakeCaller answers a long tool as the build modules do: at once, with the step and its key. +type fakeCaller struct { + calls []call + answer string + err error +} + +func (f *fakeCaller) CallOn(node, key string, body any) (json.RawMessage, error) { + f.calls = append(f.calls, call{node, key, body}) + if f.err != nil { + return nil, f.err + } + return json.RawMessage(f.answer), nil +} + +// fakeSteps holds the steps values by `/`; Wait on a key it does not hold blocks until the context ends. +type fakeSteps struct { + mu sync.Mutex + steps map[string]delivery.Step + // during, when set, is called as a wait starts and the function it answers as the wait ends. + during func(key string) func() +} + +func (f *fakeSteps) Wait(ctx context.Context, module, key string) (delivery.Step, error) { + f.mu.Lock() + s, ok := f.steps[module+"/"+key] + during := f.during + f.mu.Unlock() + if ok { + if during != nil { + defer during(key)() + } + return s, nil + } + <-ctx.Done() + return delivery.Step{}, context.Cause(ctx) +} + +func stepAt(node, id, state, reason string, result string) delivery.Step { + s := delivery.Step{ID: id, Tool: "build_go_fetch", Node: node, State: state, Reason: reason} + if result != "" { + s.Result = json.RawMessage(result) + } + return s +} + +func TestRunCallsTheToolOnTheNodeAndReadsTheStepsKey(t *testing.T) { // 1607/AC5 + caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`} + steps := &fakeSteps{steps: map[string]delivery.Step{"mesh-build-go/ace.abc1": stepAt("ace", "abc1", delivery.StepPassed, "", `{"workspace":"w1","commit":"c"}`)}} + res, err := Runner{Caller: caller, Steps: steps, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", map[string]any{"repository": "novox/r", "commit": "c"}) + if err != nil { + t.Fatal(err) + } + if len(caller.calls) != 1 || caller.calls[0].node != "ace" || caller.calls[0].key != "mesh-build-go.build_go_fetch" { + t.Errorf("calls = %+v, want one on ace for mesh-build-go.build_go_fetch", caller.calls) + } + var fr struct{ Workspace string } + if err := json.Unmarshal(res, &fr); err != nil || fr.Workspace != "w1" { + t.Errorf("result = %s (%v), want the step's result", res, err) + } +} + +func TestRunFailsOnAFailedOrRefusedStepNamingItsReason(t *testing.T) { // 1607/AC5 + for _, state := range []string{delivery.StepFailed, delivery.StepRefused} { + caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`} + steps := &fakeSteps{steps: map[string]delivery.Step{"mesh-build-go/ace.abc1": stepAt("ace", "abc1", state, "go is go1.26.0, the recipe builds with go1.27.1", "")}} + _, err := Runner{Caller: caller, Steps: steps, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_build", nil) + if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), state) || !strings.Contains(err.Error(), "ace") { + t.Errorf("%s: error = %v, want the node, the state and the reason", state, err) + } + } +} + +func TestRunStopsWaitingAtTheBound(t *testing.T) { // 1607/AC5 + caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`} + _, err := Runner{Caller: caller, Steps: &fakeSteps{}, Bound: 20 * time.Millisecond}.Run(context.Background(), "mesh-build-go", "ace", "build_go_build", nil) + if err == nil || !strings.Contains(err.Error(), "ace.abc1") || !strings.Contains(err.Error(), "did not end") { + t.Errorf("error = %v, want the step key and that it did not end within its bound", err) + } +} + +func TestRunRefusesAnAnswerWithNoKeyAndAKeyOfAnotherNode(t *testing.T) { // 1607/AC5 + for _, answer := range []string{`{"state":"running"}`, `{"step":"abc1","key":"novox.abc1","state":"running"}`, `not json`} { + _, err := Runner{Caller: &fakeCaller{answer: answer}, Steps: &fakeSteps{}, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", nil) + if err == nil { + t.Errorf("answer %s: no error", answer) + } + } +} + +func TestRunReturnsTheCallsOwnError(t *testing.T) { // 1607/AC5 + boom := errors.New("no responders") + _, err := Runner{Caller: &fakeCaller{err: boom}, Steps: &fakeSteps{}, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", nil) + if !errors.Is(err, boom) || !strings.Contains(err.Error(), "ace") { + t.Errorf("error = %v, want the call's error naming the node", err) + } +} diff --git a/modules/mesh-delivery-builder/buildrun/toolchains.go b/modules/mesh-delivery-builder/buildrun/toolchains.go new file mode 100644 index 0000000..6170425 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/toolchains.go @@ -0,0 +1,64 @@ +package buildrun + +import ( + "sync" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// LanguageOf is the language each build module's toolchains state holds (the indexer's ToolchainModules, reversed). +var LanguageOf = map[string]string{GoBuildModule: "go", NodeBuildModule: "typescript"} + +// ToolchainSet holds what the build modules' toolchains buckets hold: each node's entry of a language, as the watches +// put them in. Safe for use from the watches and from a build at once. +type ToolchainSet struct { + mu sync.Mutex + by map[string]map[string]delivery.Toolchain // language -> node -> entry +} + +// Put takes a change of a build module's toolchains bucket in: a put of a node's entry (key .) or its +// delete. It answers whether the language's entries changed (a node's version or recipe, or a node appearing or going). +// A key or a value that is not an entry of the module's language is left out. +func (s *ToolchainSet) Put(module, key string, value []byte, deleted bool) bool { + language, ok := LanguageOf[module] + if !ok { + return false + } + node, keyLanguage, err := delivery.ParseToolchainKey(key) + if err != nil || keyLanguage != language { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + if deleted { + if _, had := s.by[language][node]; !had { + return false + } + delete(s.by[language], node) + return true + } + tc, err := delivery.DecodeToolchain(value) + if err != nil || tc.Language != language { + return false + } + if s.by == nil { + s.by = map[string]map[string]delivery.Toolchain{} + } + if s.by[language] == nil { + s.by[language] = map[string]delivery.Toolchain{} + } + cur, had := s.by[language][node] + s.by[language][node] = tc + return !had || cur.Version != tc.Version || cur.Recipe != tc.Recipe +} + +// Entries are the nodes' entries of a language, in no particular order. +func (s *ToolchainSet) Entries(language string) []delivery.NodeToolchain { + s.mu.Lock() + defer s.mu.Unlock() + var out []delivery.NodeToolchain + for node, tc := range s.by[language] { + out = append(out, delivery.NodeToolchain{Node: node, Toolchain: tc}) + } + return out +} diff --git a/modules/mesh-delivery-builder/buildrun/toolchains_test.go b/modules/mesh-delivery-builder/buildrun/toolchains_test.go new file mode 100644 index 0000000..4df0d38 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/toolchains_test.go @@ -0,0 +1,49 @@ +package buildrun + +import ( + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func tcValue(t *testing.T, language, version string) []byte { + t.Helper() + b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: "1", At: time.Now()}) + if err != nil { + t.Fatal(err) + } + return b +} + +func TestToolchainSetKeepsEachNodesEntryByLanguageAndSaysWhenOneChanged(t *testing.T) { // 1607/AC6 + var s ToolchainSet + if !s.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.1"), false) { + t.Error("a first entry did not count as a change") + } + if s.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.1"), false) { + t.Error("the same entry again counted as a change") + } + s.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.1"), false) + s.Put(NodeBuildModule, "ace.typescript", tcValue(t, "typescript", "tsc5"), false) + if got := s.Entries("go"); len(got) != 2 { + t.Errorf("go entries = %+v, want two", got) + } + if got := s.Entries("typescript"); len(got) != 1 || got[0].Node != "ace" || got[0].Version != "tsc5" { + t.Errorf("typescript entries = %+v", got) + } + if !s.Put(GoBuildModule, "ace.go", nil, true) || len(s.Entries("go")) != 1 { + t.Error("a delete did not remove the node's entry") + } + // What is not a toolchain of the module's language is left out. + s = ToolchainSet{} + if s.Put(GoBuildModule, "novox.typescript", tcValue(t, "typescript", "x"), false) { + t.Error("a typescript toolchain in the go build module's bucket counted") + } + if s.Put(GoBuildModule, "novox.go", []byte("not json"), false) || s.Put("other-module", "novox.go", tcValue(t, "go", "x"), false) { + t.Error("garbage or an unknown module counted") + } + if len(s.Entries("go")) != 0 { + t.Error("an entry was kept") + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index 49f5ddd..c8a31c3 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -14,9 +14,9 @@ import ( "sync" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "mesh-delivery-builder/internal/build" ) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 54bf5fa..2c11888 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -18,7 +18,7 @@ import ( "syscall" "time" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go index 407c6ba..08d5fb7 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go @@ -12,8 +12,8 @@ import ( "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // 1945 (a part exits at start for a bucket it works without): a bus that refuses every toolchains bucket leaves the diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go index 58ae514..38a9259 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go @@ -8,7 +8,7 @@ import ( "strings" "sync" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/process" "github.com/nats-io/nats.go/jetstream" diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index 9fe6b50..6451820 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -9,6 +9,7 @@ require ( ) require ( + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun v0.0.0-00010101000000-000000000000 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect github.com/klauspost/compress v1.20.0 // indirect @@ -20,3 +21,5 @@ require ( golang.org/x/sys v0.48.0 // indirect golang.org/x/time v0.15.0 // indirect ) + +replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ./buildrun diff --git a/modules/mesh-delivery-builder/internal/build/askagain.go b/modules/mesh-delivery-builder/internal/build/askagain.go index 0a7e2a9..5a4b4d1 100644 --- a/modules/mesh-delivery-builder/internal/build/askagain.go +++ b/modules/mesh-delivery-builder/internal/build/askagain.go @@ -11,9 +11,9 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up diff --git a/modules/mesh-delivery-builder/internal/build/buildagain.go b/modules/mesh-delivery-builder/internal/build/buildagain.go index 8aa6287..62d3dee 100644 --- a/modules/mesh-delivery-builder/internal/build/buildagain.go +++ b/modules/mesh-delivery-builder/internal/build/buildagain.go @@ -7,8 +7,8 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // **A failed key is built again by hand** (novox/hq issue 1126 note 72; ADR 0429 clause 18: "a fingerprint is built again diff --git a/modules/mesh-delivery-builder/internal/build/candidates.go b/modules/mesh-delivery-builder/internal/build/candidates.go index cbc1bb4..fd9cc4d 100644 --- a/modules/mesh-delivery-builder/internal/build/candidates.go +++ b/modules/mesh-delivery-builder/internal/build/candidates.go @@ -5,8 +5,8 @@ import ( "fmt" "sort" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // How a candidate's fingerprint is built (novox/hq ADR 0484 clause 3, issue 1684): the indexer writes, for each module a diff --git a/modules/mesh-delivery-builder/internal/build/claim.go b/modules/mesh-delivery-builder/internal/build/claim.go index d58be02..50b933a 100644 --- a/modules/mesh-delivery-builder/internal/build/claim.go +++ b/modules/mesh-delivery-builder/internal/build/claim.go @@ -8,9 +8,9 @@ import ( "sort" "strings" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // ErrConflict is what a Store answers when a Create finds the key or an Update finds another revision than the one given. diff --git a/modules/mesh-delivery-builder/internal/build/imports_test.go b/modules/mesh-delivery-builder/internal/build/imports_test.go new file mode 100644 index 0000000..58f6342 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/imports_test.go @@ -0,0 +1,36 @@ +package build + +import ( + "encoding/json" + "strings" + "sync" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// ADR 0593 §2 (F3): the imports a Go build module answers at a build's end reach the done key and the log's end entry, +// where the indexer keeps them on the module. +func TestTheImportsABuildAnswersReachTheDoneKeyAndTheLog(t *testing.T) { + r := newRig(t, goBundle) + imports := []string{"git.novox.be/novox/mesh-sdk/go", "git.novox.be/novox/mesh-sdk/go/announce"} + r.asks.Answer = func(a delivery.StepAsk) delivery.Step { + step := r.answer(a) + var answer map[string]any + if err := json.Unmarshal(step.Result, &answer); err == nil && a.Language == "go" { + answer["imports"] = imports + step.Result, _ = json.Marshal(answer) + } + return step + } + var logged []delivery.BuildLogEntry + var mu sync.Mutex + r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { mu.Lock(); logged = append(logged, e); mu.Unlock() } + v := r.consider(t, "modules/m") + if v.State != delivery.BuildDone || strings.Join(v.Imports, ",") != strings.Join(imports, ",") { + t.Fatalf("done key %+v, want the imports", v) + } + if len(logged) != 2 || strings.Join(logged[1].Imports, ",") != strings.Join(imports, ",") || len(logged[0].Imports) != 0 { + t.Fatalf("log %+v, want the imports on the end alone", logged) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/plan.go b/modules/mesh-delivery-builder/internal/build/plan.go index 8c83b36..4a5969a 100644 --- a/modules/mesh-delivery-builder/internal/build/plan.go +++ b/modules/mesh-delivery-builder/internal/build/plan.go @@ -8,9 +8,9 @@ import ( "sort" "strings" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // ImageBuildModule is the build module that builds an image artifact (novox/hq ADR 0487): buildah, rootless, on the nodes that diff --git a/modules/mesh-delivery-builder/internal/build/plan_test.go b/modules/mesh-delivery-builder/internal/build/plan_test.go index 4a70662..05608e9 100644 --- a/modules/mesh-delivery-builder/internal/build/plan_test.go +++ b/modules/mesh-delivery-builder/internal/build/plan_test.go @@ -3,8 +3,8 @@ package build import ( "errors" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "strings" "testing" diff --git a/modules/mesh-delivery-builder/internal/build/round4_test.go b/modules/mesh-delivery-builder/internal/build/round4_test.go index e396831..1f1673e 100644 --- a/modules/mesh-delivery-builder/internal/build/round4_test.go +++ b/modules/mesh-delivery-builder/internal/build/round4_test.go @@ -2,7 +2,7 @@ package build import ( "context" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "testing" "git.novox.be/novox/mesh-sdk/go/catalogue" diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index ed12024..e01bdce 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -6,16 +6,17 @@ import ( "errors" "fmt" "maps" + "slices" "sort" "strconv" "strings" "sync" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet). @@ -533,6 +534,7 @@ type lane struct { toolchain *delivery.ToolchainUsed digests map[string]string // artifact -> digest references map[string]string // artifact -> the reference an image was pushed to (a trunk build only) + imports []string // the mesh packages the lane's programs import, as its build module answered them err error } @@ -666,7 +668,8 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p if len(used) == 0 { used = nil } - done := delivery.BuilderBuild{State: delivery.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used} + done := delivery.BuilderBuild{State: delivery.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used, + Imports: importsOf(lanes)} if _, err := delivery.EncodeBuilderBuild(done); err != nil { // A reference the build module answered that does not name the artifact and its digest is no reference. return failed("%s%v", delivery.NotStored, err) @@ -866,6 +869,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del l.err = fmt.Errorf("the ask of %s on %s answered what is not a build: %s", l.module, step.Node, step.Result) return } + l.imports = answer.Imports built := map[string]delivery.AskBuilt{} for _, a := range answer.Artifacts { built[a.Artifact] = a @@ -950,9 +954,26 @@ func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, re e.Against = append([]string(nil), m.Inputs2.Bases...) } e.Made = append([]delivery.BuiltArtifact(nil), result.Artifacts...) + e.Imports = append([]string(nil), result.Imports...) // the indexer keeps them on the module (novox/hq ADR 0593 §2) return e } +// importsOf are the mesh packages the lanes' programs import, as their build modules answered them (novox/hq ADR 0593 +// §2): sorted and each once; nil when none answered any. A lane kept from a prior build answered none, so a build that +// kept one reports only what it built. +func importsOf(lanes []*lane) []string { + seen := map[string]bool{} + for _, l := range lanes { + for _, p := range l.imports { + seen[p] = true + } + } + if len(seen) == 0 { + return nil + } + return slices.Sorted(maps.Keys(seen)) +} + // SittingOutWords says, per node that is not on the agreed toolchain of a language, that it sits out and which version // it has against the agreed one: "ace sits out: typescript 26.12.1, agreed 26.13.0" (novox/hq ADR 0584 clause 2). func SittingOutWords(language string, a delivery.Agreement) []string { diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 6950dc3..a0bfe47 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -13,11 +13,11 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // buildModules simulates the two build modules on the nodes: a long tool answers a step and key at once and writes the diff --git a/modules/mesh-delivery-builder/internal/build/stuck.go b/modules/mesh-delivery-builder/internal/build/stuck.go index 72528fe..d6a713c 100644 --- a/modules/mesh-delivery-builder/internal/build/stuck.go +++ b/modules/mesh-delivery-builder/internal/build/stuck.go @@ -11,8 +11,8 @@ import ( "sync/atomic" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go index 33045f3..5397e72 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go @@ -10,9 +10,9 @@ import ( "sync/atomic" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "mesh-delivery-checker/internal/check" ) diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go index 03db849..c1018c8 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go @@ -21,10 +21,10 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-checker/internal/check" diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index bee45f7..d66feae 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -11,8 +11,8 @@ import ( "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-checker/internal/check" diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go index 53c5ed0..7942122 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go @@ -9,8 +9,8 @@ import ( "strings" "testing" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) type memManifests map[string][]byte diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index ded10d8..7961a97 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -20,9 +20,9 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-checker/internal/check" diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/refused_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/refused_test.go index 3c20242..9ba279c 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/refused_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/refused_test.go @@ -12,7 +12,7 @@ import ( "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" ) // 1945 (a part exits at start for a bucket it works without): a bus that refuses every toolchains bucket leaves the diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go index 99c3087..bfce12e 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go @@ -14,7 +14,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" ) // toolchainStreams are the stream each build module carries its toolchain.reported event on, as the bus names it diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index 8149fbc..d8a8fe7 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -9,6 +9,7 @@ require ( ) require ( + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun v0.0.0-00010101000000-000000000000 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect github.com/klauspost/compress v1.20.0 // indirect @@ -20,3 +21,5 @@ require ( golang.org/x/sys v0.48.0 // indirect golang.org/x/time v0.15.0 // indirect ) + +replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ../mesh-delivery-builder/buildrun diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index 628870a..bae8e4b 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -16,10 +16,10 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/composer" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // captured reads a result the build module's own code marshalled (testdata/captured/README). diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index ba16aaf..23fc775 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -11,9 +11,9 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // goModule is the build module the Go repositories are judged by. diff --git a/modules/mesh-delivery-checker/internal/check/lostask.go b/modules/mesh-delivery-checker/internal/check/lostask.go index 8328e46..bc1db98 100644 --- a/modules/mesh-delivery-checker/internal/check/lostask.go +++ b/modules/mesh-delivery-checker/internal/check/lostask.go @@ -3,9 +3,9 @@ package check import ( "strings" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // **An ask the build seat gave up on ends its step** (novox/hq issue 734): mesh-sdk buildrun.LostAskWaiter hands the diff --git a/modules/mesh-delivery-checker/internal/check/lostask_test.go b/modules/mesh-delivery-checker/internal/check/lostask_test.go index ddf4d33..9a1581c 100644 --- a/modules/mesh-delivery-checker/internal/check/lostask_test.go +++ b/modules/mesh-delivery-checker/internal/check/lostask_test.go @@ -8,9 +8,9 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // losingAsks is the build seat over a bus that gives up the first `lose` asks put (DEAD_LETTERS keeps them, as on diff --git a/modules/mesh-delivery-checker/internal/check/read.go b/modules/mesh-delivery-checker/internal/check/read.go index 30cb4d5..7997547 100644 --- a/modules/mesh-delivery-checker/internal/check/read.go +++ b/modules/mesh-delivery-checker/internal/check/read.go @@ -7,8 +7,8 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the diff --git a/modules/mesh-delivery-checker/internal/check/try.go b/modules/mesh-delivery-checker/internal/check/try.go index 757ce4e..990cf68 100644 --- a/modules/mesh-delivery-checker/internal/check/try.go +++ b/modules/mesh-delivery-checker/internal/check/try.go @@ -13,8 +13,8 @@ import ( "sync" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // A try is a pushed head judged by the checker as a pull request's head is, before its pull request is opened (ADR 0501 diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 0e56eac..4a12dd0 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -23,9 +23,9 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/internal/index" diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 3f506f6..03277d0 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -9,6 +9,7 @@ require ( ) require ( + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun v0.0.0-00010101000000-000000000000 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect github.com/klauspost/compress v1.20.0 // indirect @@ -20,3 +21,5 @@ require ( golang.org/x/sys v0.48.0 // indirect golang.org/x/time v0.15.0 // indirect ) + +replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ../mesh-delivery-builder/buildrun diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index 5fe76b0..e32291c 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -9,6 +9,7 @@ require ( ) require ( + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun v0.0.0-00010101000000-000000000000 github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect github.com/klauspost/compress v1.20.1 // indirect @@ -20,3 +21,5 @@ require ( golang.org/x/sys v0.48.0 // indirect golang.org/x/time v0.16.0 // indirect ) + +replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ../mesh-delivery-builder/buildrun