From c623e8d235f437388b29f4de835737c63bd27f40 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 25 Oct 2026 21:43:08 +0100 Subject: [PATCH 1/4] Builder: ask again an ask its stream can no longer answer, and give it tools to read and ask again (issue 1945) A build ask its seat's stream gave up on, dropped or ended with no step was waited for with no end, and on restart taken over and waited for again: no build node would ever answer it, and nothing could ask it again (trunk key of mesh-build-node at 4e7b118e, building since 19:06; seven keys at 18:44). At the step bound, a context's bound and no timer of its own, the builder reads what the bus says of an ask with no step: gone from its stream, or first in line while a node took a later ask. Such an ask is asked again under a new id the key names, logged and said as build.asked-again, at most three askings, then the key fails with why. The bus's max-deliveries advisory would tell it sooner, but the builder is granted neither it nor the workers' information, and the controller is not changed here. builder_stuck reads every waiting key with its asks, askings and steps; builder_ask_again writes the new reasks state, which the builder watches and answers. --- .../cmd/mesh-delivery-builder-tools/main.go | 92 +++++++ .../cmd/mesh-delivery-builder/held.go | 81 ++++-- .../cmd/mesh-delivery-builder/main.go | 31 ++- .../cmd/mesh-delivery-builder/reasks.go | 75 ++++++ .../internal/build/askagain.go | 235 ++++++++++++++++++ .../internal/build/run.go | 22 +- .../internal/build/run_test.go | 205 +++++++++++++-- .../internal/build/stuck.go | 200 +++++++++++++++ .../mesh-delivery-builder/manifest_test.go | 37 ++- modules/mesh-delivery-builder/module.json | 21 +- 10 files changed, 948 insertions(+), 51 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go create mode 100644 modules/mesh-delivery-builder/internal/build/askagain.go create mode 100644 modules/mesh-delivery-builder/internal/build/stuck.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go new file mode 100644 index 0000000..852cb29 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -0,0 +1,92 @@ +// Command mesh-delivery-builder-tools serves the builder's tools (novox/hq issue 1945): the runtime launches it for a call. +// builder_stuck reads the builder's builds and reasks states and the build modules' steps through the builder's state and +// reads; builder_ask_again writes one value of the reasks state, which the builder watches and answers. The builder is the +// other program: it serves no tool (ADR 0437 §4). +package main + +import ( + "encoding/json" + "fmt" + "os" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-builder/internal/build" +) + +// Tools are the builder's two tools: stuck reads, askAgain writes the reasks value. +func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, now func() time.Time) []stdio.Tool { + str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + return []stdio.Tool{ + {Name: build.StuckTool, + Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " + + "waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " + + "answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " + + "seat workers' delivery counts are not readable by the builder. With module, that module's keys only. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "only this module's keys"}}}, + Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }}, + {Name: build.AskAgainTool, + Description: "Ask a waiting build key's asks again now, under new ids the key then names, as the builder does by itself for an ask its build " + + "seat's stream gave up on. The builder answers in the reasks state, which " + build.StuckTool + " shows beside the key; a key no ask of " + + "which waits is answered so and nothing is asked.", + Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module's name"}, + "fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:, as builder_stuck shows it"}, + "why": map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + key, value, err := build.NewReAsk(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now()) + if err != nil { + return nil, err + } + if err := askAgain(key, value); err != nil { + return nil, fmt.Errorf("the ask could not be written to %s: %w", build.ReAsksBucket, err) + } + return map[string]any{"key": key, "asked": true, + "next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil + }}, + } +} + +func read(s stdio.KeptState) build.StateRead { + return build.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, time.Time, error) { + e, err := s.Get(key) + if err != nil || e == nil { + return nil, time.Time{}, err + } + return e.Value, written(s, key, e.Revision), nil + }} +} + +// written is when the key's revision was written, from its history, or zero. +func written(s stdio.KeptState, key string, revision uint64) time.Time { + h, err := s.History(key) + if err != nil { + return time.Time{} + } + for _, r := range h { + if r.Revision == revision { + return r.At + } + } + return time.Time{} +} + +func main() { + steps := map[string]build.StateRead{} + for _, bm := range build.StuckBuildModules { + steps[bm] = read(stdio.State(bm + ".steps")) + } + reasks := stdio.State(build.ReAsksState) + r := build.StuckReader{Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps} + put := func(key string, value []byte) error { + _, err := reasks.Put(key, json.RawMessage(value)) + return err + } + if err := stdio.Serve("", Tools(r, put, time.Now)); err != nil { + fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err) + os.Exit(1) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go index e5493a4..1c5afd8 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go @@ -8,57 +8,82 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-builder/internal/build" ) // heldOnTheBus says whether an ask a build key names is on its build seat's stream, or a build node wrote a step for it -// (issue 1910): read once, at the take-over of a key at start, never polled. +// (issue 1910), read once at the take-over of a key at start; and, at the builder's check bound, whether an ask can still +// be answered (issue 1945). Never polled. type heldOnTheBus struct{ js jetstream.JetStream } // Held is build.Builder's Held: a step key `.` in the build module's steps, or a message of the stream under // the ask's subject whose Nats-Msg-Id is the ask's id. func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error) { - module, err := delivery.BuildModuleOf(a.Language) + seen, err := h.seen(ctx, a, false) if err != nil { return false, err } + return seen.Step || seen.OnStream, nil +} + +// Gone is build.Builder's Gone: why the ask can no longer be answered (build.AskSeen.Why), or "". +func (h heldOnTheBus) Gone(ctx context.Context, a delivery.StepAsk) (string, error) { + seen, err := h.seen(ctx, a, true) + if err != nil { + return "", err + } + return seen.Why(), nil +} + +// seen reads what the bus holds of the ask; with later, a step of its build module started after the ask was put too, +// when the ask waits first in line. +func (h heldOnTheBus) seen(ctx context.Context, a delivery.StepAsk, later bool) (build.AskSeen, error) { + var seen build.AskSeen + module, err := delivery.BuildModuleOf(a.Language) + if err != nil { + return seen, err + } pattern, err := delivery.AskStepWatch(a.ID) if err != nil { - return false, err + return seen, err } steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module)) if err != nil { - return false, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err) + return seen, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err) } keys, err := steps.ListKeysFiltered(ctx, pattern) if err != nil { - return false, err + return seen, err } for range keys.Keys() { keys.Stop() - return true, nil // a node took it + seen.Step = true // a node took it + return seen, nil } subject, err := delivery.AskSubject(a.Language, a.Kind) if err != nil { - return false, err + return seen, err } // The seat's stream by its name (delivery.AskStream), as the broker grants the builder its information and its // messages by sequence and nothing wider (issue 1910). name, err := delivery.AskStream(a.Language) if err != nil { - return false, err + return seen, err } + seen.Stream = name stream, err := h.js.Stream(ctx, name) if errors.Is(err, jetstream.ErrStreamNotFound) { - return false, nil + return seen, nil } if err != nil { - return false, err + return seen, err } // A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read by sequence // instead, each a direct read that consumes nothing. info, err := stream.Info(ctx) if err != nil { - return false, err + return seen, err } for seq := info.State.FirstSeq; seq != 0 && seq <= info.State.LastSeq; seq++ { m, err := stream.GetMsg(ctx, seq) @@ -66,11 +91,37 @@ func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error continue // taken and acknowledged, or removed } if err != nil { - return false, err + return seen, err } - if m.Subject == subject && m.Header.Get(jetstream.MsgIDHeader) == a.ID { - return true, nil + if m.Subject != subject { + continue } + if m.Header.Get(jetstream.MsgIDHeader) == a.ID { + seen.OnStream, seen.PutAt = true, m.Time + break + } + seen.Ahead++ } - return false, nil + if !later || !seen.OnStream || seen.Ahead > 0 { + return seen, nil + } + // First in line and not taken: did a build node take any ask put after it? Its stream then no longer delivers it. + all, err := steps.ListKeys(ctx) + if err != nil { + return seen, err + } + defer all.Stop() + for k := range all.Keys() { + e, err := steps.Get(ctx, k) + if err != nil { + continue + } + s, err := delivery.DecodeStep(e.Value()) + if err != nil || !s.Started.After(seen.PutAt) { + continue // not a step (a node's pause key), or one started before the ask was put + } + seen.Later = &s + break + } + return seen, nil } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 77cbc22..3d867ce 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -169,9 +169,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } defer calls.Close() b := &build.Builder{ - Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module}, - Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound}, - Held: heldOnTheBus{js}, + Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module}, + Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound}, + Held: heldOnTheBus{js}, + // An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound. + Gone: heldOnTheBus{js}, + CheckAfter: bound, Toolchains: tools, Manifests: manifestReader{manifests}, Logf: logger.Printf, @@ -198,6 +201,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } b.Trunk = trunks.head } + b.Said = func(e build.AskedAgain) { + if err := emitAskedAgain(context.WithoutCancel(ctx), js, cred, e); err != nil { + logger.Printf("%v", err) + } + } // **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the // controller delivers from this builder's keys only while an instance says it writes references. A bucket not there // yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent @@ -290,8 +298,23 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } + // A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as + // every other input. A bucket not there yet (the manifest's state not made) is said. + errs := make(chan error, 4) + switch kv, err := js.KeyValue(ctx, build.ReAsksBucket); { + case errors.Is(err, jetstream.ErrBucketNotFound): + logger.Printf("the bucket %s does not exist yet: builder_ask_again is not heard until the builder starts again", build.ReAsksBucket) + case err != nil: + return fmt.Errorf("the bucket %s: %w", build.ReAsksBucket, err) + default: + eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) } + rw, err := settle(ctx, kv, eachReAsk) + if err != nil { + return err + } + go func() { errs <- follow(ctx, rw, eachReAsk) }() + } logger.Printf("started on %s", cred.Node) - errs := make(chan error, 3) go func() { errs <- follow(ctx, w, each) }() go func() { errs <- follow(ctx, cw, eachCandidate) }() if sw != nil { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go new file mode 100644 index 0000000..48bcd29 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go @@ -0,0 +1,75 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "mesh-delivery-builder/internal/build" +) + +// askAgainer is the part of the builder a reasks value asks through. +type askAgainer interface { + AskAgain(key, why string) []string +} + +// reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the +// key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value +// answered already (this builder's own write) or deleted is left. +func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) { + if e.Operation() != jetstream.KeyValuePut { + return + } + r, err := build.DecodeReAsk(e.Value()) + if err != nil { + logf("%s.%s holds what is not an ask to ask again, left: %v", build.ReAsksState, e.Key(), err) + return + } + if r.Answer != "" { + return + } + asked := b.AskAgain(e.Key(), r.Why) + if len(asked) == 0 { + r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands" + } else { + r.Answer = "asked again: the " + strings.Join(asked, ", ") + " ask, under a new id the key names" + } + r.Answered = time.Now().UTC() + logf("%s: asked again by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer) + data, err := json.Marshal(r) + if err != nil { + return + } + if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err) + } +} + +// emitAskedAgain says the builder's event build.asked-again (issue 1945): mesh.mod..event.build.asked-again, +// with the SDK's envelope headers and the new ask's id as the bus's dedup id. +func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred credential, e build.AskedAgain) error { + data, err := json.Marshal(e) + if err != nil { + return err + } + msg := nats.NewMsg("mesh.mod." + cred.Module + ".event." + build.AskedAgainEvent) + id := "asked-again-" + e.New + msg.Header.Set("x-event-id", id) + msg.Header.Set("x-source", cred.Module) + msg.Header.Set("x-node", cred.Node) + msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Data = data + pub, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil { + return fmt.Errorf("saying %s for %s: %w", build.AskedAgainEvent, e.Key, err) + } + return nil +} diff --git a/modules/mesh-delivery-builder/internal/build/askagain.go b/modules/mesh-delivery-builder/internal/build/askagain.go new file mode 100644 index 0000000..980b5f6 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/askagain.go @@ -0,0 +1,235 @@ +package build + +import ( + "context" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up +// once it was delivered as often as its worker may (delivery.AskMaxDeliver), and a message given up on, dropped or +// terminated with no step written is answered by no build node: the lane waiting for its step would wait for ever, and on +// restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound), +// a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new +// id, written into the key, said in its log and as the event build.asked-again, at most Askings times; then the key is +// failed with why. A person asks the same by hand with the tool builder_ask_again (AskAgain). + +// DefaultAskings is how many times a build module's part of a build is asked, the first included, when Builder.Askings is 0. +const DefaultAskings = 3 + +// AskedAgainEvent is the event the builder says when it asked an ask again: mesh.mod.mesh-delivery-builder.event.. +const AskedAgainEvent = "build.asked-again" + +// AskedAgain is the body of AskedAgainEvent. +type AskedAgain struct { + Key string `json:"key"` + Module string `json:"module"` + FP string `json:"fp"` + BuildModule string `json:"build_module"` + Language string `json:"language"` + Old string `json:"old"` + New string `json:"new"` + Asking int `json:"asking"` + Askings int `json:"askings"` + Why string `json:"why"` + At time.Time `json:"at"` +} + +// AskSeen is what the bus says of an ask whose step has not ended, as the builder may read it: its build seat's stream +// (information and messages by sequence) and its build module's steps. The builder may not read the seat workers' +// delivery counts (granted to the delivery view alone), so a message given up on is told by what the workers did after it. +type AskSeen struct { + Stream string + // Step is whether a build node wrote a step for the ask (running: the step bound of buildrun.Asker judges it). + Step bool + // OnStream is whether the stream still holds the ask's message; PutAt is when it was put, Ahead how many messages of + // its subject wait before it. + OnStream bool + PutAt time.Time + Ahead int + // Later is a step of the same build module that started after the ask was put, when one did. + Later *delivery.Step +} + +// Why is why the ask can no longer be answered, or "" when it can: it has a step, waits behind others of its kind, or no +// build node took any ask since it was put (the nodes are busy or paused: a backlog, never a fault, ADR 0525 §9). +func (s AskSeen) Why() string { + switch { + case s.Step: + return "" + case !s.OnStream: + return fmt.Sprintf("no build node wrote a step for it and %s no longer holds it: it was given up on or dropped", s.Stream) + case s.Ahead > 0 || s.Later == nil: + return "" + default: + return fmt.Sprintf("it waits first in line on %s since %s while %s took a later ask at %s: its stream no longer delivers it (given up after its most deliveries)", + s.Stream, s.PutAt.UTC().Format(time.RFC3339), s.Later.Node, s.Later.Started.UTC().Format(time.RFC3339)) + } +} + +func (b *Builder) askings() int { + if b.Askings > 0 { + return b.Askings + } + return DefaultAskings +} + +var askingWords = regexp.MustCompile(`asked (\S+) again \(asking (\d+)\)`) + +// askingOf is the asking a key's step words name for a build module's ask, 1 when none. +func askingOf(step, buildModule string) int { + for _, m := range askingWords.FindAllStringSubmatch(step, -1) { + if m[1] == buildModule { + if n, err := strconv.Atoi(m[2]); err == nil && n > 0 { + return n + } + } + } + return 1 +} + +// AskAgain asks the waiting asks of a build key again now, under new ids (the tool builder_ask_again): it answers the +// languages whose lane waits, none when no lane of this builder waits on the key. +func (b *Builder) AskAgain(key, why string) []string { + b.init() + b.mu.Lock() + defer b.mu.Unlock() + var asked []string + for language, c := range b.waiting[key] { + select { + case c <- why: + default: // one is already asked + } + asked = append(asked, language) + } + return asked +} + +// Waiting are the build keys whose lanes wait for a step now, each with its languages. +func (b *Builder) Waiting() map[string][]string { + b.init() + b.mu.Lock() + defer b.mu.Unlock() + out := map[string][]string{} + for key, ls := range b.waiting { + for l := range ls { + out[key] = append(out[key], l) + } + } + return out +} + +func (b *Builder) listen(key, language string) chan string { + b.init() + c := make(chan string, 1) + b.mu.Lock() + if b.waiting[key] == nil { + b.waiting[key] = map[string]chan string{} + } + b.waiting[key][language] = c + b.mu.Unlock() + return c +} + +func (b *Builder) unlisten(key, language string, c chan string) { + b.mu.Lock() + if b.waiting[key][language] == c { + delete(b.waiting[key], language) + if len(b.waiting[key]) == 0 { + delete(b.waiting, key) + } + } + b.mu.Unlock() +} + +var ( + errCheck = errors.New("the check bound passed") + errAgain = errors.New("asked again") +) + +type byHand struct{ why string } + +func (h byHand) Error() string { return "asked again by hand: " + h.why } + +// answer waits for the step of the lane's ask, and asks it again while it can no longer be answered (Gone) or a person +// asked it again (AskAgain); it answers the step and the ask that has it. +func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, asked *askNotes, module, fp string) (delivery.Step, delivery.StepAsk, error) { + key, _ := delivery.BuilderBuildKey(module, fp) + again := b.listen(key, ask.Language) + defer b.unlisten(key, ask.Language, again) + asking := askingOf(asked.step, l.module) + for { + step, why, err := b.waitOnce(ctx, ask, again) + switch { + case errors.Is(err, errCheck): + continue + case errors.Is(err, errAgain): + if asking >= b.askings() { + return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why) + } + asking++ + old := ask.ID + ask.ID, ask.At = buildstep.NewID(), time.Now() + if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("asked %s again (asking %d): %s", l.module, asking, why)); err != nil { + return step, ask, fmt.Errorf("the ask of %s asked again could not be written to the key: %w", l.module, err) + } + if err := b.Asks.Put(ctx, ask); err != nil { + return step, ask, err + } + b.logf("%s: the ask %s of %s is asked again as %s (asking %d of %d): %s", key, old, l.module, ask.ID, asking, b.askings(), why) + if b.Said != nil { + b.Said(AskedAgain{Key: key, Module: module, FP: fp, BuildModule: l.module, Language: ask.Language, Old: old, New: ask.ID, + Asking: asking, Askings: b.askings(), Why: why, At: ask.At}) + } + default: + return step, ask, err + } + } +} + +// waitOnce waits for the ask's step until it ends, a person asks it again, or the check bound passes: then it asks Gone +// and answers errAgain with why when the ask can no longer be answered, errCheck when it waits on. +func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-chan string) (delivery.Step, string, error) { + hand, cancel := context.WithCancelCause(ctx) + defer cancel(nil) + go func() { + select { + case why := <-again: + cancel(byHand{why}) + case <-hand.Done(): + } + }() + wait := hand + if b.Gone != nil && b.CheckAfter > 0 { + var stop context.CancelFunc + wait, stop = context.WithTimeoutCause(hand, b.CheckAfter, errCheck) + defer stop() + } + step, err := b.Asks.Wait(wait, ask) + if err == nil || ctx.Err() != nil { + return step, "", err + } + var h byHand + if errors.As(context.Cause(hand), &h) { + return step, strings.TrimSpace("asked again by hand: " + h.why), errAgain + } + if errors.Is(context.Cause(wait), errCheck) { + why, gerr := b.Gone.Gone(ctx, ask) + if gerr != nil { + b.logf("whether the ask %s can still be answered cannot be read, so it is waited for: %v", ask.ID, gerr) + return step, "", errCheck + } + if why == "" { + return step, "", errCheck + } + return step, why, errAgain + } + return step, "", err +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 492cae5..6283da8 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -56,6 +56,18 @@ type Builder struct { // Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a // candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why. Trunk func(ctx context.Context, rev uint64, repository string) (string, error) + // Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the + // ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked + // again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it. + Gone interface { + Gone(ctx context.Context, a delivery.StepAsk) (string, error) + } + CheckAfter time.Duration + // Askings is how many times one build module's part of a build is asked, the first included, before its key is failed + // with why; DefaultAskings when 0. + Askings int + // Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody. + Said func(AskedAgain) once sync.Once sem chan struct{} @@ -75,6 +87,8 @@ type Builder struct { // held are the done keys' references the store answered held, by ` `, asked once per process: a // blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937). held map[string]bool + // waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them. + waiting map[string]map[string]chan string } type planned struct { @@ -91,7 +105,7 @@ func (b *Builder) init() { b.sem = make(chan struct{}, limit) b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{} b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{} - b.published, b.held = map[string]bool{}, map[string]bool{} + b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{} }) } @@ -673,12 +687,14 @@ type askNotes struct { b *Builder cl *Claim resumed map[string]string + // step is the step words of a key taken over at start: an ask asked again says its asking there (askingOf). + step string } func (b *Builder) notes(cl *Claim) *askNotes { n := &askNotes{b: b, cl: cl} if cl.Build.State == delivery.BuildBuilding && len(cl.Build.Asks) > 0 { - n.resumed = maps.Clone(cl.Build.Asks) + n.resumed, n.step = maps.Clone(cl.Build.Asks), cl.Build.Step } return n } @@ -776,7 +792,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del return } } - step, err := b.Asks.Wait(ctx, ask) + step, ask, err := b.answer(ctx, l, ask, asked, module, m.FP2) if err != nil { l.err = fmt.Errorf("%s: %w", l.module, err) return diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index f4ad7b2..17d5cf4 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -616,27 +616,198 @@ func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // re } } -// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked -// again; the key is done when a node takes it. -func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { - r := newRig(t, goBundle) - r.asks.Answer = nil - r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond} - r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) - for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) == 0; time.Sleep(time.Millisecond) { +// goneAsks answers Gone from a function of the ask, and counts how often it was asked. +type goneAsks struct { + mu sync.Mutex + n int + why func(a delivery.StepAsk) string +} + +func (g *goneAsks) Gone(_ context.Context, a delivery.StepAsk) (string, error) { + g.mu.Lock() + g.n++ + g.mu.Unlock() + return g.why(a), nil +} + +func (g *goneAsks) asked() int { + g.mu.Lock() + defer g.mu.Unlock() + return g.n +} + +// waitForAsks waits until n asks were put. +func waitForAsks(t *testing.T, r *rig, n int) []delivery.StepAsk { + t.Helper() + for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) < n; time.Sleep(time.Millisecond) { if time.Now().After(deadline) { - t.Fatal("no ask was put") + t.Fatalf("%d asks put, want %d", len(r.asks.Asks()), n) } } - time.Sleep(100 * time.Millisecond) - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding { - t.Fatalf("the key = %+v while its ask waits, want building", v) - } - r.asks.End(r.answer(r.asks.Asks()[0])) - r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 { - t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks())) + return r.asks.Asks() +} + +// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked +// again; the key is done when a node takes it. Issue 1945: an ask that can no longer be answered (its stream gave it up) +// is asked again under a new id, said, at most Askings times; a person asks it again by hand the same way. +func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { + t.Run("waits in its stream: never asked again", func(t *testing.T) { + r := newRig(t, goBundle) + r.asks.Answer = nil + r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond} + gone := &goneAsks{why: func(delivery.StepAsk) string { return "" }} + r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + waitForAsks(t, r, 1) + time.Sleep(100 * time.Millisecond) + if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding { + t.Fatalf("the key = %+v while its ask waits, want building", v) + } + if gone.asked() == 0 { + t.Fatal("whether the ask can still be answered was never read at the check bound") // 1945 + } + r.asks.End(r.answer(r.asks.Asks()[0])) + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 { + t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks())) + } + }) + for _, c := range []struct { + name string + askings int + // step is the key's step words at start when it names an ask (a key taken over), "" for a fresh build. + step string + byHand bool + always bool // Gone says every ask cannot be answered, not only the first + puts int + done bool + asking int // the asking the event says, 0 when none is said + }{ + {name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2}, + {name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2}, + {name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2}, + {name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0}, + } { + t.Run(c.name, func(t *testing.T) { // 1945 + r := newRig(t, goBundle) + r.asks.Answer = nil + r.b.Askings = c.askings + var said []AskedAgain + var mu sync.Mutex + r.b.Said = func(e AskedAgain) { mu.Lock(); said = append(said, e); mu.Unlock() } + first := "" + if c.step != "" { + first = "mvl0frv18d1960" + data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD, + Step: c.step, Asks: map[string]string{delivery.AskGo: first}}) + r.st.vals[r.key] = data + } + if !c.byHand { + r.b.Gone = &goneAsks{why: func(a delivery.StepAsk) string { + if c.always || a.ID == first || first == "" { + return "it waits first in line while a node took a later ask" + } + return "" + }} + r.b.CheckAfter = 5 * time.Millisecond + } + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + if c.byHand { + waitForAsks(t, r, 1) + for deadline := time.Now().Add(2 * time.Second); len(r.b.AskAgain(r.key, "stuck since 19:06")) == 0; time.Sleep(time.Millisecond) { + if time.Now().After(deadline) { + t.Fatal("no lane of the key waits") + } + } + } + if c.done { + asks := waitForAsks(t, r, c.puts) + if first == "" && !c.byHand { + first = asks[0].ID + } + last := asks[len(asks)-1] + if v := valueOf(t, r.st, r.key); v.Asks[delivery.AskGo] != last.ID || !strings.Contains(v.Step, "again (asking 2)") { + t.Fatalf("the key = %+v, want building naming the ask asked again %s", v, last.ID) + } + r.asks.End(r.answer(last)) + } + r.b.Wait() + v := valueOf(t, r.st, r.key) + if n := len(r.asks.Asks()); n != c.puts { + t.Errorf("%d asks put, want %d", n, c.puts) + } + if c.done && v.State != delivery.BuildDone { + t.Errorf("the key = %+v, want done from the ask asked again", v) + } + if !c.done && (v.State != delivery.BuildFailed || !strings.Contains(v.Error, "could not be answered and was asked")) { + t.Errorf("the key = %+v, want failed saying the ask could not be answered", v) + } + mu.Lock() + defer mu.Unlock() + if c.asking > 0 && (len(said) != 1 || said[0].Asking != c.asking || said[0].New == said[0].Old || said[0].Key != r.key || said[0].Why == "") { + t.Errorf("said %+v, want once at asking %d", said, c.asking) + } + }) } + t.Run("what the bus says of an ask", func(t *testing.T) { // 1945 + put := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC) + later := &delivery.Step{Node: "g14", Started: put.Add(time.Minute)} + for _, c := range []struct { + seen AskSeen + want string // a part of the why, "" when it can be answered + }{ + {AskSeen{Stream: "S", Step: true}, ""}, + {AskSeen{Stream: "S"}, "no longer holds it"}, + {AskSeen{Stream: "S", OnStream: true, PutAt: put, Later: later}, "no longer delivers it"}, + {AskSeen{Stream: "S", OnStream: true, PutAt: put, Ahead: 2, Later: later}, ""}, + {AskSeen{Stream: "S", OnStream: true, PutAt: put}, ""}, + } { + got := c.seen.Why() + if (c.want == "") != (got == "") || !strings.Contains(got, c.want) { + t.Errorf("%+v: why %q, want %q", c.seen, got, c.want) + } + } + }) + t.Run("the tool builder_stuck", func(t *testing.T) { // 1945 + at := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC) + state := func(vals map[string]string) StateRead { + return StateRead{Keys: func() ([]string, error) { + var ks []string + for k := range vals { + ks = append(ks, k) + } + return ks, nil + }, Get: func(k string) (json.RawMessage, time.Time, error) { + if v, ok := vals[k]; ok { + return json.RawMessage(v), at, nil + } + return nil, time.Time{}, nil + }} + } + key, _ := delivery.BuilderBuildKey("m", fp1) + done, _ := delivery.BuilderBuildKey("d", fp1) + building, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD, + Step: "asked mesh-build-go again (asking 2): x", Asks: map[string]string{delivery.AskGo: "mvo53jcm70b5f7", delivery.AskImage: "mvo53jcm70b5f8"}}) + finished, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: manifestD, Step: "built"}) + running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at}) + _, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at) + r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}), + Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})}, + Now: func() time.Time { return at.Add(time.Hour) }} + out, err := r.Stuck("") + if err != nil { + t.Fatal(err) + } + stuck := out["stuck"].([]Stuck) + if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" { + t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck) + } + a := stuck[0].Asks + if len(a) != 2 || a[0].Language != delivery.AskGo || a[0].Asking != 2 || a[0].Step != "none" || a[0].Asked.IsZero() || + a[1].Asking != 1 || !strings.HasPrefix(a[1].Step, "running on g14") { + t.Fatalf("asks = %+v, want go at asking 2 with no step and image at asking 1 running on g14", a) + } + }) } // heldAsks says an ask is held when its id is among held. diff --git a/modules/mesh-delivery-builder/internal/build/stuck.go b/modules/mesh-delivery-builder/internal/build/stuck.go new file mode 100644 index 0000000..5bcaf0c --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/stuck.go @@ -0,0 +1,200 @@ +package build + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" +) + +// The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads +// every key that waits on an ask, builder_ask_again asks a key's build again now. The tools bundle serves them through +// the tool runner; the builder's process serves no tool (ADR 0437 §4), so builder_ask_again writes the ask into the +// builder's state reasks, which the builder watches and answers there. +const ( + StuckTool = "builder_stuck" + AskAgainTool = "builder_ask_again" + // ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key. + ReAsksState = "reasks" + ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState +) + +// ReAsk is a value of the reasks state: why a person asked the key's build again, who and when, and, once the builder +// read it, its answer. +type ReAsk struct { + V int `json:"v"` + Why string `json:"why"` + By string `json:"by,omitempty"` + At time.Time `json:"at"` + Answer string `json:"answer,omitempty"` + Answered time.Time `json:"answered,omitzero"` +} + +// DecodeReAsk reads a reasks value. +func DecodeReAsk(data []byte) (ReAsk, error) { + var r ReAsk + if err := json.Unmarshal(data, &r); err != nil { + return r, err + } + if r.V != 1 { + return r, fmt.Errorf("a reasks value of version %d, not 1", r.V) + } + return r, nil +} + +// NewReAsk is the value builder_ask_again writes for a module's fingerprint: its key and the value. +func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) { + key, err := delivery.BuilderBuildKey(module, fp) + if err != nil { + return "", nil, err + } + if strings.TrimSpace(why) == "" { + return "", nil, errors.New("say why the build is asked again (why)") + } + data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC()}) + return key, data, err +} + +// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known). +type StateRead struct { + Keys func() ([]string, error) + Get func(key string) (json.RawMessage, time.Time, error) +} + +// StuckAsk is one ask a key waits on. +type StuckAsk struct { + Language string `json:"language"` + BuildModule string `json:"build_module"` + ID string `json:"id"` + Asking int `json:"asking"` + // Asked is when the ask was put, read from its id; Step is what its build module's steps hold of it: none, or its + // state on its node since its start. + Asked time.Time `json:"asked,omitzero"` + Step string `json:"step"` +} + +// Stuck is one build key that waits on an ask. +type Stuck struct { + Key string `json:"key"` + Module string `json:"module"` + FP string `json:"fp"` + State string `json:"state"` + Step string `json:"step"` + By string `json:"by"` + Since time.Time `json:"since,omitzero"` + Age string `json:"age,omitempty"` + Asks []StuckAsk `json:"asks"` + ReAsk *ReAsk `json:"reask,omitempty"` + Unasked bool `json:"unasked,omitempty"` + NoneStep int `json:"asks_with_no_step"` +} + +// StuckReader reads what builder_stuck answers. +type StuckReader struct { + Builds StateRead + ReAsks StateRead + // Steps are the build modules' steps states, by build module. + Steps map[string]StateRead + Now func() time.Time +} + +// Stuck is every key building or queued with its asks, oldest first; with module, that module's only. +func (r StuckReader) Stuck(module string) (map[string]any, error) { + keys, err := r.Builds.Keys() + if err != nil { + return nil, err + } + now := time.Now + if r.Now != nil { + now = r.Now + } + stepKeys := map[string][]string{} + for bm, s := range r.Steps { + ks, err := s.Keys() + if err != nil { + return nil, fmt.Errorf("the steps of %s: %w", bm, err) + } + stepKeys[bm] = ks + } + out := []Stuck{} + for _, key := range keys { + mod, fp, err := delivery.ParseBuilderBuildKey(key) + if err != nil || (module != "" && mod != module) { + continue + } + raw, at, err := r.Builds.Get(key) + if err != nil || raw == nil { + continue + } + v, err := delivery.DecodeBuilderBuild(raw) + if err != nil || (v.State != delivery.BuildBuilding && v.State != delivery.BuildQueued) { + continue + } + s := Stuck{Key: key, Module: mod, FP: fp, State: v.State, Step: v.Step, By: v.By, Since: at, Asks: []StuckAsk{}} + if !at.IsZero() { + s.Age = now().Sub(at).Round(time.Second).String() + } + for language, id := range v.Asks { + bm, _ := delivery.BuildModuleOf(language) + a := StuckAsk{Language: language, BuildModule: bm, ID: id, Asking: askingOf(v.Step, bm), Step: "none"} + if t, ok := idTime(id); ok { + a.Asked = t + } + if st, ok := r.Steps[bm]; ok { + for _, k := range stepKeys[bm] { + if !strings.HasSuffix(k, "."+id) { + continue + } + if raw, _, err := st.Get(k); err == nil && raw != nil { + if step, err := delivery.DecodeStep(raw); err == nil { + a.Step = fmt.Sprintf("%s on %s since %s", step.State, step.Node, step.Started.UTC().Format(time.RFC3339)) + } + } + } + } + if a.Step == "none" { + s.NoneStep++ + } + s.Asks = append(s.Asks, a) + } + sort.Slice(s.Asks, func(i, j int) bool { return s.Asks[i].Language < s.Asks[j].Language }) + s.Unasked = v.State == delivery.BuildBuilding && len(v.Asks) == 0 + if r.ReAsks.Get != nil { + if raw, _, err := r.ReAsks.Get(key); err == nil && raw != nil { + if ra, err := DecodeReAsk(raw); err == nil { + s.ReAsk = &ra + } + } + } + out = append(out, s) + } + sort.Slice(out, func(i, j int) bool { + if !out[i].Since.Equal(out[j].Since) { + return out[i].Since.Before(out[j].Since) + } + return out[i].Key < out[j].Key + }) + return map[string]any{"stuck": out, "count": len(out), + "delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}, nil +} + +// StuckBuildModules are the build modules whose steps builder_stuck reads. +var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule} + +// idTime is when a step id was made (buildstep.NewID: the time in milliseconds in base 36, then six hex digits). +func idTime(id string) (time.Time, bool) { + if len(id) <= 6 { + return time.Time{}, false + } + ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64) + if err != nil || ms <= 0 { + return time.Time{}, false + } + return time.UnixMilli(ms).UTC(), true +} diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 27e44e9..7034836 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -23,18 +23,21 @@ type state struct { } type artifact struct { - Name string `json:"name"` - Kind string `json:"kind"` - Language string `json:"language"` - System string `json:"system"` - From string `json:"from"` - Binary string `json:"binary"` + Name string `json:"name"` + Kind string `json:"kind"` + Language string `json:"language"` + System string `json:"system"` + From string `json:"from"` + Binary string `json:"binary"` + Loads []string `json:"loads"` } type manifest struct { Module string `json:"module"` Version string `json:"version"` Slug string `json:"slug"` + Tools []string `json:"tools"` + Emits []string `json:"emits"` Consumes []string `json:"consumes"` State []state `json:"state"` Reads []string `json:"reads"` @@ -84,7 +87,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if m.Module != "mesh-delivery-builder" { t.Errorf("module = %q", m.Module) } - if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory + if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory t.Errorf("state = %+v, want %+v", m.State, want) } // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and @@ -112,7 +115,14 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if m.SecretsOwner != "mesh-delivery-builder" { t.Errorf("secrets-owner = %q", m.SecretsOwner) } - for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "defines-seats", "tools", "seat", "seats", "bus", "config"} { + // 1945: the builder's tools, served by its tools bundle, and the event it says when it asks an ask again. + if want := []string{"builder_ask_again", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) { + t.Errorf("tools = %v, want %v", m.Tools, want) + } + if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) { + t.Errorf("emits = %v, want %v", m.Emits, want) + } + for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the builder has none", k) } @@ -127,7 +137,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1 procs = append(procs, r) } if r["type"] == "tool" || r["type"] == "seat" { - t.Errorf("resource %v: the builder serves no tool and holds no seat", r) + t.Errorf("resource %v: the builder's tools are its bundle's, and it holds no seat", r) } } if len(procs) != 1 { @@ -149,8 +159,13 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1 if _, bus := p["bus"]; bus { t.Errorf("the process declares a bus: it runs on its own broker credential") } - if len(m.Build.Artifacts) != 1 { - t.Fatalf("artifacts = %+v, want the program alone", m.Build.Artifacts) + if len(m.Build.Artifacts) != 2 { + t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts) + } + if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-builder-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-builder-tools"}) { + t.Errorf("tools artifact = %+v", a) // 1945 + } else if st, err := os.Stat(a.From); err != nil || !st.IsDir() { + t.Errorf("the tools' source %s: %v", a.From, err) } a := m.Build.Artifacts[0] if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-builder" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index e15c633..7d62e68 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -2,6 +2,13 @@ "module": "mesh-delivery-builder", "version": "1", "slug": "builder", + "tools": [ + "builder_ask_again", + "builder_stuck" + ], + "emits": [ + "build.asked-again" + ], "consumes": [ "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", @@ -12,7 +19,8 @@ ], "state": [ { "name": "builds", "history": 5 }, - { "name": "instances", "history": 1 } + { "name": "instances", "history": 1 }, + { "name": "reasks", "history": 1 } ], "reads": [ "mesh-build-go.steps", @@ -78,6 +86,17 @@ "system": "arch", "from": "cmd/mesh-delivery-builder", "binary": "mesh-delivery-builder" + }, + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-delivery-builder-tools", + "binary": "mesh-delivery-builder-tools", + "loads": [ + "mesh-delivery-builder-tools" + ] } ] } From d4a054bfed3a058882799fedb7c6acfb906c3569 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 25 Oct 2026 21:45:43 +0100 Subject: [PATCH 2/4] Assigner: say why a node gets no want, in the journal and in its removed want (issue 1945) A new assignment whose trunk build was not done fell through the rules with nothing said: no want, no neither-list entry, no log line, and the rule holding it back was computed and dropped. The pass now logs the rule when a node's reason appears or changes and when it has a want again, one line per change. A node whose want is removed and that does not run the module gets the reason in that removed want's words, so assigner_want answers why; a node with no key gets no removed want, since its node agent would then remove the module and stop holding its resources back. --- .../cmd/mesh-delivery-assigner/pass.go | 41 +++++++++++++++++++ .../cmd/mesh-delivery-assigner/pass_test.go | 25 +++++++++++ .../internal/assign/wants.go | 17 +++++++- .../internal/assign/wants_test.go | 27 ++++++++++++ 4 files changed, 109 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 1bbe91f..32a1dcc 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "fmt" "strings" "time" @@ -35,6 +36,9 @@ type passer struct { neither map[string]neitherOne // acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket). acts *putActs + // noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is + // said in the journal when it appears or changes, and its clearing when the node has a want again; one line per change. + noWant map[string]string } func (p *passer) clock() time.Time { @@ -148,6 +152,7 @@ func (p *passer) wants(ctx context.Context) { } out := assign.Wants(m, nodes, wanted[name], last) gatherNeither(neither, name, out) + p.sayNoWant(name, m.Nodes, wanted[name], out) if first { startClears(stale, m, out) if fp != "" && !m.Held && out.Held == nil && !out.Lifted { @@ -186,6 +191,42 @@ func (p *passer) wants(ctx context.Context) { } } +// sayNoWant says in the journal why a node of module gets no want after this pass, from the rule the outcome names for it +// (issue 1945: a new assignment whose trunk build was not done was held back with nothing said), once per reason, and says +// when the node has a want again. A node with a live want, written before and kept or written in this pass, has one. +func (p *passer) sayNoWant(module string, nodes []string, before map[string]delivery.Want, out assign.Outcome) { + if p.noWant == nil { + p.noWant = map[string]string{} + } + assigned := map[string]bool{} + for _, node := range nodes { + assigned[node] = true + key := node + "." + module + w, written := out.Wants[node] + if !written { + w, written = before[node] + } + reason := "" + if h, held := out.Rules[node]; held && (!written || w.Removed) { + reason = fmt.Sprintf("rule %d: %s", h.Rule, h.Words) + } + last, said := p.noWant[key] + switch { + case reason != "" && reason != last: + p.logf("%s on %s gets no want: %s", module, node, reason) + p.noWant[key] = reason + case reason == "" && said && written && !w.Removed: + p.logf("%s on %s has a want", module, node) + delete(p.noWant, key) + } + } + for key := range p.noWant { + if node, m, _ := strings.Cut(key, "."); m == module && !assigned[node] { + delete(p.noWant, key) // no longer assigned there: its removed want says so + } + } +} + // neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back. type neitherOne struct { node, module string diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 0458ebc..ff438f1 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -380,6 +380,31 @@ func TestThePassGathersTheModulesOnNeitherList(t *testing.T) { if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant { t.Errorf("gathered = %+v", got) } + // 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and + // when it has a want again: one line per change, not per pass. + p := newPasser(buckets{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC)) + var logged []string + p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) } + notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleBuildDone, Words: "the build of rofi at 4e7b118e is not done"}}} + wanted := assign.Outcome{Wants: map[string]delivery.Want{"shanks": {FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}}} + for i, c := range []struct { + before map[string]delivery.Want + out assign.Outcome + line string // the line said in this pass, "" for none + }{ + {nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is not done"}, + {nil, notDone, ""}, + {map[string]delivery.Want{"shanks": {Removed: true}}, notDone, ""}, + {nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"}, + {nil, wanted, "rofi on shanks has a want"}, + {map[string]delivery.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""}, + } { + logged = nil + p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out) + if got := strings.Join(logged, "\n"); got != c.line { + t.Errorf("pass %d said %q, want %q", i, got, c.line) + } + } } // A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index 8d0daff..e275c59 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -333,8 +333,18 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st // the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2, // held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither // list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour). - if h, there := n.Haves[node][m.Name]; there && h.Phase != delivery.PhaseRemoved && !held { + h, there := n.Haves[node][m.Name] + switch { + case there && h.Phase != delivery.PhaseRemoved && !held: out.Neither[node] = Holding{Rule: rule, Words: words} + case had && (!there || h.Phase == delivery.PhaseRemoved): + // A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's + // assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was + // would have its node agent remove the module and stop holding its composition's resources back. + next = delivery.Want{Removed: true, Words: NoWantWords(rule, words)} + if !delivery.SameWant(normal(before), normal(next)) { + out.Wants[node] = next + } } continue } @@ -370,6 +380,11 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st return out } +// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945). +func NoWantWords(rule int, words string) string { + return fmt.Sprintf("no want: rule %d: %s", rule, words) +} + // failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or // moving). A failed health while it // backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index 69deb19..25ae9d1 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -867,6 +867,33 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) { if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone { t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither) } + // 1945 (the assigner holds a first want back silently): with no build, a node whose want was removed and that does not run + // the module has the reason written into its removed want, once; a node with no key gets no want written, and a node that + // runs the module is on neither list, its removed want left as it is. + gone := delivery.Want{Removed: true, Words: "screen-lock is no longer assigned to shanks"} + for _, c := range []struct { + name string + before map[string]delivery.Want + want map[string]delivery.Want + }{ + {"no key: nothing written", nil, map[string]delivery.Want{}}, + {"a removed want: it says why", map[string]delivery.Want{"shanks": gone}, + map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}}, + {"a removed want that says why already: nothing written", map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}, + map[string]delivery.Want{}}, + {"a node that runs it: on neither list, its removed want left", map[string]delivery.Want{"g14": gone}, map[string]delivery.Want{}}, + } { + out := Wants(m, n, c.before, nil) + if len(out.Wants) != len(c.want) { + t.Errorf("%s: wants = %+v, want %+v", c.name, out.Wants, c.want) + continue + } + for node, w := range c.want { + if !delivery.SameWant(normal(out.Wants[node]), w) { + t.Errorf("%s: want of %s = %+v, want %+v", c.name, node, out.Wants[node], w) + } + } + } m.FP = "" if out := Wants(m, n, nil, nil); len(out.Neither) != 0 { t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither) From 20027e53ff773ca0beb9fee5cb16fc59f630e79d Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 25 Oct 2026 22:00:50 +0100 Subject: [PATCH 3/4] Delivery parts: a bucket the bus refuses at start stops a part only when it cannot work without it (issue 1945) Met twice on 2026-10-25: the builder of #48 exited on a refused read of the gate's source and crash-looped, and its new reasks bucket would have done the same. A grant reaches the bus with a delivery, so a refusal or a timeout at start is often only a grant not there yet; exiting for it stopped the whole part until a hand step on the bus. Every startup open in the parts is now essential or optional. Essential ones (a part's own state and the input it exists for) still stop the start, with an error naming the bucket and its grant. Optional ones are said and left: watches the bus refuses too. The builder builds trunk keys without the gate's source, which it opens again at the next candidate that asks a trunk head; toolchains and previews open again at their next event, as a missing bucket did. builder_ask_again says that asking a running step again builds it twice. --- .../cmd/mesh-delivery-assigner/main.go | 6 +- .../cmd/mesh-delivery-assigner/main_test.go | 2 +- .../cmd/mesh-delivery-assigner/optional.go | 27 +++++ .../cmd/mesh-delivery-builder-tools/main.go | 3 +- .../cmd/mesh-delivery-builder/main.go | 91 +++++---------- .../cmd/mesh-delivery-builder/optional.go | 104 ++++++++++++++++++ .../cmd/mesh-delivery-builder/standing.go | 4 +- .../mesh-delivery-builder/standing_test.go | 61 ++++++++++ .../cmd/mesh-delivery-builder/toolchains.go | 8 +- .../cmd/mesh-delivery-checker/main.go | 69 +++++------- .../cmd/mesh-delivery-checker/optional.go | 41 +++++++ .../cmd/mesh-delivery-checker/standing.go | 4 +- .../mesh-delivery-checker/standing_test.go | 51 +++++++++ .../cmd/mesh-delivery-checker/toolchains.go | 8 +- .../cmd/mesh-delivery-forge-bridge/main.go | 23 ++-- .../mesh-delivery-forge-bridge/main_test.go | 16 +++ .../mesh-delivery-forge-bridge/optional.go | 46 ++++++++ .../cmd/mesh-delivery-indexer/main.go | 50 ++++----- .../cmd/mesh-delivery-indexer/optional.go | 41 +++++++ .../cmd/mesh-delivery-indexer/previews.go | 8 +- .../cmd/mesh-delivery-indexer/standing.go | 4 +- .../cmd/mesh-delivery-indexer/toolchains.go | 8 +- .../mesh-delivery-indexer/toolchains_test.go | 11 ++ .../cmd/mesh-delivery-teller/main.go | 21 ++-- .../cmd/mesh-delivery-teller/optional.go | 46 ++++++++ .../cmd/mesh-delivery-teller/teller_test.go | 18 +++ .../cmd/mesh-delivery-watchman/main.go | 53 +++++---- .../cmd/mesh-delivery-watchman/main_test.go | 19 ++++ .../cmd/mesh-delivery-watchman/optional.go | 46 ++++++++ 29 files changed, 681 insertions(+), 208 deletions(-) create mode 100644 modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/optional.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/optional.go create mode 100644 modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/optional.go create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/optional.go create mode 100644 modules/mesh-delivery-teller/cmd/mesh-delivery-teller/optional.go create mode 100644 modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/optional.go diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 0c7341d..2f2512d 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -86,11 +86,11 @@ var keptStates = []announce.State{ // buckets is every bucket the assigner opened, by name. type buckets map[string]jetstream.KeyValue -// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named, since a pass +// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named with its grant (issue 1945), since a pass // without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it // does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants // without it and takes no put further until it starts again (ADR 0546). -func open(ctx context.Context, js jetstream.JetStream, logf func(string, ...any)) (buckets, error) { +func open(ctx context.Context, js bucketOpener, logf func(string, ...any)) (buckets, error) { out := buckets{} for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) { kv, err := js.KeyValue(ctx, name) @@ -99,7 +99,7 @@ func open(ctx context.Context, js jetstream.JetStream, logf func(string, ...any) continue } if err != nil { - return nil, fmt.Errorf("the bucket %s cannot be opened: %w", name, err) + return nil, essentialErr(name, "write a want from the whole mesh", err) } out[name] = kv } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go index 3c346ab..75d7411 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go @@ -73,7 +73,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) { switch { case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")): t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged) - case !c.starts && (err == nil || !strings.Contains(err.Error(), c.missing)): + case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err) } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/optional.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/optional.go new file mode 100644 index 0000000..0267144 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/optional.go @@ -0,0 +1,27 @@ +package main + +import ( + "context" + "fmt" + + "github.com/nats-io/nats.go/jetstream" +) + +// **A part does not stop for a bucket it can work without** (novox/hq issue 1945, met twice on 2026-10-25: the builder of +// mesh-delivery #48 exited on a refused read of the gate's source, and its new reasks bucket would have done the same). A +// grant reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted +// yet: exiting for it stops the part until a person's hand. The assigner works without none of its buckets but the Go build +// module's steps, so the others stop the start, naming the grant to look at. (The same two functions are in each delivery part; +// they belong in mesh-sdk.) + +// bucketOpener is the part of the bus a start opens buckets with. +type bucketOpener interface { + KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error) +} + +// essentialErr is the error of a bucket the part cannot work without: it names the bucket and the grant. +func essentialErr(name, does string, err error) error { + return fmt.Errorf("the bucket %s cannot be read (%v): the assigner cannot %s without it, so it stops; where the bus "+ + "refused it or did not answer, this module's grant of %s (its manifest's reads or state, issued with its credential) "+ + "has not reached the bus yet", name, err, does, name) +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index 852cb29..acc2bfe 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -31,7 +31,8 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, n {Name: build.AskAgainTool, Description: "Ask a waiting build key's asks again now, under new ids the key then names, as the builder does by itself for an ask its build " + "seat's stream gave up on. The builder answers in the reasks state, which " + build.StuckTool + " shows beside the key; a key no ask of " + - "which waits is answered so and nothing is asked.", + "which waits is answered so and nothing is asked. An ask whose step already runs on a node is asked again too: it is then " + + "built twice, the first run's work wasted and its end read by nobody, never a wrong key; builder_stuck shows whether a step runs.", Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{ "module": map[string]any{"type": "string", "description": "the module's name"}, "fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:, as builder_stuck shows it"}, diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 3d867ce..d6b2e9f 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -124,29 +124,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - bucket := func(name string) (jetstream.KeyValue, error) { - kv, err := js.KeyValue(ctx, name) - if err != nil { - return nil, fmt.Errorf("the bucket %s: %w", name, err) - } - return kv, nil - } - builds, err := bucket(delivery.BuilderBuildsBucket) + builds, err := essentialBucket(ctx, js, delivery.BuilderBuildsBucket, "claim or write a build key") if err != nil { return err } - modules, err := bucket(delivery.ModulesBucket) + modules, err := essentialBucket(ctx, js, delivery.ModulesBucket, "know any fingerprint to build") if err != nil { return err } - manifests, err := bucket(delivery.ManifestsBucket) - if err != nil { - return err - } - candidates, err := bucket(delivery.CandidatesBucket) + manifests, err := essentialBucket(ctx, js, delivery.ManifestsBucket, "read what a module builds") if err != nil { return err } + // The candidates' fingerprints are built too (issue 1684): a builder that cannot read them builds the trunk's. + candidates := optionalBucket(ctx, js, delivery.CandidatesBucket, "no candidate's fingerprint is built until the builder starts again; the trunk's are", logger.Printf) bound := defaultStepBound if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" { if bound, err = time.ParseDuration(v); err != nil { @@ -181,26 +172,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Stored: storedVia(calls.Call), } // A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base - // revision (the gate's source state). Where the bucket is not there yet no candidate's fingerprint is built, and said. - // The source's revisions are watched with their history (a direct get of one revision is not what a read grants): a - // revision heard after the candidate that names it builds that candidate then. - trunks := &sourceHeads{} - var sw jetstream.KeyWatcher - switch kv, err := js.KeyValue(ctx, delivery.SourceBucket); { - case errors.Is(err, jetstream.ErrBucketNotFound): - logger.Printf("the bucket %s does not exist yet: no candidate's fingerprint is built until the builder starts again", delivery.SourceBucket) - case err != nil: - return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err) - default: - w, err := kv.Watch(ctx, delivery.SourceMainKey, jetstream.IncludeHistory()) - if err != nil { - return fmt.Errorf("the bucket %s cannot be watched: %w", delivery.SourceBucket, err) - } - if sw, err = settled(ctx, w, func(e jetstream.KeyValueEntry) { trunks.put(e) }); err != nil { - return err - } - b.Trunk = trunks.head - } + // revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision + // is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it + // cannot be read the trunk's keys are built and the candidates wait (issue 1945). + errs := make(chan error, 4) + trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: b.TrunkMoved, errs: errs, logf: logger.Printf} + trunks.start() + b.Trunk = trunks.head b.Said = func(e build.AskedAgain) { if err := emitAskedAgain(context.WithoutCancel(ctx), js, cred, e); err != nil { logger.Printf("%v", err) @@ -210,11 +188,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // controller delivers from this builder's keys only while an instance says it writes references. A bucket not there // yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent // is asked since R4g, ADR 0524 §7). - switch kv, err := js.KeyValue(ctx, instancesBucket); { - case errors.Is(err, jetstream.ErrBucketNotFound): - logger.Printf("the bucket %s does not exist yet: the controller is not told that this builder writes references until it starts again", instancesBucket) - case err != nil: - return fmt.Errorf("the bucket %s: %w", instancesBucket, err) + switch kv := optionalBucket(ctx, js, instancesBucket, "the controller is not told that this builder writes references until it starts again", logger.Printf); { + case kv == nil: default: says := newSayer(ctx, kv, cred.Node, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, build.ImageBuildModule}, logger.Printf) b.Answered = says.Answered @@ -294,38 +269,26 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro b.ForgetCandidate(e.Key()) } } - cw, err := settle(ctx, candidates, eachCandidate) - if err != nil { - return err + if candidates != nil { + cw, err := settle(ctx, candidates, eachCandidate) + if err != nil { + logger.Printf("the bucket %s cannot be watched (%v): no candidate's fingerprint is built until the builder starts again", delivery.CandidatesBucket, err) + } else { + go func() { errs <- follow(ctx, cw, eachCandidate) }() + } } // A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as - // every other input. A bucket not there yet (the manifest's state not made) is said. - errs := make(chan error, 4) - switch kv, err := js.KeyValue(ctx, build.ReAsksBucket); { - case errors.Is(err, jetstream.ErrBucketNotFound): - logger.Printf("the bucket %s does not exist yet: builder_ask_again is not heard until the builder starts again", build.ReAsksBucket) - case err != nil: - return fmt.Errorf("the bucket %s: %w", build.ReAsksBucket, err) - default: + // every other input. The builder asks again by itself without it. + if kv := optionalBucket(ctx, js, build.ReAsksBucket, "builder_ask_again is not heard until the builder starts again; an ask that can no longer be answered is still asked again by itself", logger.Printf); kv != nil { eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) } - rw, err := settle(ctx, kv, eachReAsk) - if err != nil { - return err + if rw, err := settle(ctx, kv, eachReAsk); err != nil { + logger.Printf("the bucket %s cannot be watched (%v): builder_ask_again is not heard until the builder starts again", build.ReAsksBucket, err) + } else { + go func() { errs <- follow(ctx, rw, eachReAsk) }() } - go func() { errs <- follow(ctx, rw, eachReAsk) }() } logger.Printf("started on %s", cred.Node) go func() { errs <- follow(ctx, w, each) }() - go func() { errs <- follow(ctx, cw, eachCandidate) }() - if sw != nil { - go func() { - errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { - if trunks.put(e) { - b.TrunkMoved(ctx) - } - }) - }() - } select { case <-ctx.Done(): b.Wait() diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go new file mode 100644 index 0000000..bd2542e --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go @@ -0,0 +1,104 @@ +package main + +import ( + "context" + "fmt" + "sync" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// **A part does not stop for a bucket it can work without** (novox/hq issue 1945, met twice on 2026-10-25: the builder of +// mesh-delivery #48 exited on a refused read of the gate's source, and its reasks bucket would have done the same). A grant +// reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted yet: +// exiting for it stops every build until a person's hand. A bucket the builder cannot build at all without (its builds, +// the indexer's modules and manifests) still stops the start, naming the grant to look at; every other is said and left, +// and opened again at the next event that needs it where there is one. + +// essentialBucket opens a bucket the builder cannot work without; its error names the bucket and the grant. +func essentialBucket(ctx context.Context, js bucketOpener, name, does string) (jetstream.KeyValue, error) { + kv, err := js.KeyValue(ctx, name) + if err != nil { + return nil, essentialErr(name, does, err) + } + return kv, nil +} + +// essentialErr is the error of a bucket the part cannot work without: it names the bucket and the grant. +func essentialErr(name, does string, err error) error { + return fmt.Errorf("the bucket %s cannot be read (%v): the builder cannot %s without it, so it stops; where the bus "+ + "refused it or did not answer, this module's grant of %s (its manifest's reads or state, issued with its credential) "+ + "has not reached the bus yet", name, err, does, name) +} + +// optionalBucket opens a bucket the builder works without: any error is said with what is not heard, and nil answered. +func optionalBucket(ctx context.Context, js bucketOpener, name, without string, logf func(string, ...any)) jetstream.KeyValue { + kv, err := js.KeyValue(ctx, name) + if err != nil { + logf("the bucket %s cannot be read (%v): %s", name, err, without) + return nil + } + return kv +} + +// trunkSource is the gate's source history the candidates' previews are merged onto (sourceHeads), opened at start and, +// while it cannot be, again at the next candidate that asks a trunk head: a builder without it builds the trunk's keys, +// and a candidate's preview waits, saying why. +type trunkSource struct { + ctx context.Context + js bucketOpener + heads *sourceHeads + moved func(context.Context) + errs chan<- error + logf func(string, ...any) + + mu sync.Mutex + opened bool +} + +// ensure opens and settles the source's watch once, and follows it; an error leaves it closed for the next ask. +func (t *trunkSource) ensure() error { + t.mu.Lock() + defer t.mu.Unlock() + if t.opened { + return nil + } + kv, err := t.js.KeyValue(t.ctx, delivery.SourceBucket) + if err != nil { + return fmt.Errorf("the bucket %s cannot be read (%v)", delivery.SourceBucket, err) + } + w, err := kv.Watch(t.ctx, delivery.SourceMainKey, jetstream.IncludeHistory()) + if err != nil { + return fmt.Errorf("the bucket %s cannot be watched (%v)", delivery.SourceBucket, err) + } + sw, err := settled(t.ctx, w, func(e jetstream.KeyValueEntry) { t.heads.put(e) }) + if err != nil { + return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err) + } + t.opened = true + go func() { + t.errs <- follow(t.ctx, sw, func(e jetstream.KeyValueEntry) { + if t.heads.put(e) && t.moved != nil { + t.moved(t.ctx) + } + }) + }() + return nil +} + +// start opens the source at start; one that cannot be is said, and the builder goes on. +func (t *trunkSource) start() { + if err := t.ensure(); err != nil { + t.logf("%v: the trunk's keys are built, and a candidate's preview waits until the source can be read, tried again at its next candidate", err) + } +} + +// head is Builder.Trunk: the head of repository at source revision rev, opening the source first if it is not yet. +func (t *trunkSource) head(ctx context.Context, rev uint64, repository string) (string, error) { + if err := t.ensure(); err != nil { + return "", err + } + return t.heads.head(ctx, rev, repository) +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing.go index 3d6decf..9a06cec 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing.go @@ -193,7 +193,9 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) } w, err := settle(ctx, kv, each) if err != nil { - return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err) + // Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read. + logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err) + return errs, nil } if gate.Settled() { changed(ctx) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go index 28fccc4..ae19091 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go @@ -3,11 +3,17 @@ package main // The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940). import ( + "bytes" + "context" + "errors" "fmt" + "log" "strings" "testing" "time" + "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) @@ -89,6 +95,61 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) { if n := len(set.Entries("typescript")); n != 2 { t.Fatalf("novox and shanks are left; got %d entries", n) } + + // 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every + // toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without + // is an error naming the bucket and its grant. + var logs bytes.Buffer + logger := log.New(&logs, "", 0) + refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")} + if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil { + t.Errorf("a refused watch of the assignments stopped the start: %v", err) + } + if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil { + t.Errorf("a refused toolchains bucket stopped the start: %v", err) + } + if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") { + t.Errorf("the refusals were not said: %q", logs.String()) + } + if _, err := essentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "mesh-delivery-indexer_modules", "work"); err == nil || + !strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") { + t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err) + } + // The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk + // head is told why and opens it again then. + src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf} + src.start() + if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) { + t.Errorf("a trunk head with the source refused: %v, want why", err) + } + if src.opened { + t.Error("a refused source counts as opened, so it is never opened again") + } +} + +// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err. +type refusingBus struct { + refuseOpen bool + err error +} + +func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { + if r.refuseOpen { + return nil, r.err + } + return refusedWatch{err: r.err}, nil +} + +type refusedWatch struct { + jetstream.KeyValue + err error +} + +func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + return nil, r.err +} +func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + return nil, r.err } // A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go index 69faff2..f574c2d 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go @@ -3,7 +3,6 @@ package main import ( "context" "errors" - "fmt" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "log" "strings" @@ -62,11 +61,14 @@ func (w *toolchainWatch) ensure(module string) error { w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module) return nil case err != nil: - return fmt.Errorf("the bucket %s: %w", name, err) + // Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report. + w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module) + return nil } watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) }) if err != nil { - return err + w.logger.Printf("the bucket %s cannot be watched (%v): %s's toolchains are not heard until its next report", name, err, module) + return nil } w.watched[module] = true go func() { diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 297cfce..7b97f0d 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -152,17 +152,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - bucket := func(name string) (jetstream.KeyValue, error) { - kv, err := js.KeyValue(ctx, name) - if err != nil { - return nil, fmt.Errorf("the bucket %s: %w", name, err) - } - return kv, nil - } + // Its own states, the candidates it checks, and the modules and manifests that say which build modules judge a repository: + // a check without one of them would be wrong or unwritten, so each stops the start, naming its grant (issue 1945). buckets := map[string]jetstream.KeyValue{} for _, name := range []string{delivery.CheckerVerdictsBucket, delivery.CheckerKeptBucket, delivery.CheckerOwedBucket, delivery.PreviewsBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.ManifestsBucket, check.TriesBucket} { - if buckets[name], err = bucket(name); err != nil { + if buckets[name], err = essentialBucket(ctx, js, name, "judge or keep a check"); err != nil { return err } } @@ -212,12 +207,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line // says nothing of the builds, until the checker starts again. var builds check.BuildStates - switch kv, err := js.KeyValue(ctx, delivery.BuilderBuildsBucket); { - case errors.Is(err, jetstream.ErrBucketNotFound): - logger.Printf("the bucket %s does not exist yet: the line says nothing of the builds until the checker starts again", delivery.BuilderBuildsBucket) - case err != nil: - return fmt.Errorf("the bucket %s: %w", delivery.BuilderBuildsBucket, err) - default: + if kv := optionalBucket(ctx, js, delivery.BuilderBuildsBucket, "the line says nothing of the builds until the checker starts again", logger.Printf); kv != nil { builds = buildReader{kvStore{kv}} } // The assigner's compositions (ADR 0528 clause 5, issue 1297): their bucket is looked up at each check until it is @@ -290,47 +280,38 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // the assigner's own presence says whether a composition is being made (compositionWatch, ADR 0537 §4). A check is no // longer held for a node: its asks wait in the build seats' streams, which redeliver a lost node's ask (ADR 0525 §6, // issue 1912), so the node-engine's have is not watched here. - switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); { - case errors.Is(err, jetstream.ErrBucketNotFound): - logger.Printf("the bucket %s does not exist yet: no node counts as online until the checker starts again", delivery.PresenceBucket) - case err != nil: - return fmt.Errorf("the bucket %s: %w", delivery.PresenceBucket, err) - default: - pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) }) - if err != nil { - return err + if kv := optionalBucket(ctx, js, delivery.PresenceBucket, "no node counts as online until the checker starts again", logger.Printf); kv != nil { + if pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) }); err != nil { + logger.Printf("the bucket %s cannot be watched (%v): no node counts as online until the checker starts again", delivery.PresenceBucket, err) + } else { + go func() { + errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) { + if presence.Put(e.Key(), e.Value(), !opOf(e)) { + ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard + } + }) + }() } - go func() { - errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) { - if presence.Put(e.Key(), e.Value(), !opOf(e)) { - ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard - } - }) - }() } // The gate's source says the trunk head of each repository, which a pull request is merged onto. Where it is not there // yet no trunk is known, so nothing is checked. - switch kv, err := js.KeyValue(ctx, delivery.SourceBucket); { - case errors.Is(err, jetstream.ErrBucketNotFound): - logger.Printf("the bucket %s does not exist yet: no trunk is known until the checker starts again", delivery.SourceBucket) - case err != nil: - return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err) - default: + if kv := optionalBucket(ctx, js, delivery.SourceBucket, "no trunk is known, so nothing is checked until the checker starts again", logger.Printf); kv != nil { sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { if e.Key() == delivery.SourceMainKey { trunks.Put(e.Value(), !opOf(e)) } }) if err != nil { - return err + logger.Printf("the bucket %s cannot be watched (%v): no trunk is known, so nothing is checked until the checker starts again", delivery.SourceBucket, err) + } else { + go func() { + errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { + if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) { + ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again + } + }) + }() } - go func() { - errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { - if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) { - ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again - } - }) - }() } // The modules say which repository holds a TypeScript module, so the Node build module judges it too. mw, err := settle(ctx, buckets[delivery.ModulesBucket], func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) }) diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/optional.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/optional.go new file mode 100644 index 0000000..466cdd0 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/optional.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + "fmt" + + "github.com/nats-io/nats.go/jetstream" +) + +// **A part does not stop for a bucket it can work without** (novox/hq issue 1945, met twice on 2026-10-25: the builder of +// mesh-delivery #48 exited on a refused read of the gate's source, and its new reasks bucket would have done the same). A +// grant reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted +// yet: exiting for it stops the part until a person's hand. A bucket the checker cannot do its job at all without still stops +// the start, naming the grant to look at; every other is said and left. (The same two functions are in each delivery part; +// they belong in mesh-sdk.) + +// essentialBucket opens a bucket the checker cannot work without; its error names the bucket and the grant. +func essentialBucket(ctx context.Context, js bucketOpener, name, does string) (jetstream.KeyValue, error) { + kv, err := js.KeyValue(ctx, name) + if err != nil { + return nil, essentialErr(name, does, err) + } + return kv, nil +} + +// essentialErr is the error of a bucket the part cannot work without: it names the bucket and the grant. +func essentialErr(name, does string, err error) error { + return fmt.Errorf("the bucket %s cannot be read (%v): the checker cannot %s without it, so it stops; where the bus "+ + "refused it or did not answer, this module's grant of %s (its manifest's reads or state, issued with its credential) "+ + "has not reached the bus yet", name, err, does, name) +} + +// optionalBucket opens a bucket the checker works without: any error is said with what is not heard, and nil answered. +func optionalBucket(ctx context.Context, js bucketOpener, name, without string, logf func(string, ...any)) jetstream.KeyValue { + kv, err := js.KeyValue(ctx, name) + if err != nil { + logf("the bucket %s cannot be read (%v): %s", name, err, without) + return nil + } + return kv +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing.go index 3d6decf..9a06cec 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing.go @@ -193,7 +193,9 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) } w, err := settle(ctx, kv, each) if err != nil { - return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err) + // Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read. + logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err) + return errs, nil } if gate.Settled() { changed(ctx) diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go index 28fccc4..c759d3a 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go @@ -3,11 +3,17 @@ package main // The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940). import ( + "bytes" + "context" + "errors" "fmt" + "log" "strings" "testing" "time" + "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) @@ -89,6 +95,51 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) { if n := len(set.Entries("typescript")); n != 2 { t.Fatalf("novox and shanks are left; got %d entries", n) } + + // 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every + // toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without + // is an error naming the bucket and its grant. + var logs bytes.Buffer + logger := log.New(&logs, "", 0) + refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")} + if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil { + t.Errorf("a refused watch of the assignments stopped the start: %v", err) + } + if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil { + t.Errorf("a refused toolchains bucket stopped the start: %v", err) + } + if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") { + t.Errorf("the refusals were not said: %q", logs.String()) + } + if _, err := essentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "mesh-delivery-indexer_modules", "work"); err == nil || + !strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") { + t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err) + } +} + +// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err. +type refusingBus struct { + refuseOpen bool + err error +} + +func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { + if r.refuseOpen { + return nil, r.err + } + return refusedWatch{err: r.err}, nil +} + +type refusedWatch struct { + jetstream.KeyValue + err error +} + +func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + return nil, r.err +} +func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + return nil, r.err } // A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go index bc848ca..0fe2f41 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go @@ -3,7 +3,6 @@ package main import ( "context" "errors" - "fmt" "log" "strings" "sync" @@ -63,11 +62,14 @@ func (w *toolchainWatch) ensure(module string) error { w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module) return nil case err != nil: - return fmt.Errorf("the bucket %s: %w", name, err) + // Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report. + w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module) + return nil } watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) }) if err != nil { - return err + w.logger.Printf("the bucket %s cannot be watched (%v): %s's toolchains are not heard until its next report", name, err, module) + return nil } w.watched[module] = true go func() { diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index a11a3b2..383bedc 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -188,18 +188,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro call := callerOn(conn, cred.User) // No forge event is heard while the open pull requests are read and said (ADR 0443 §5): events and syncs take turns. var turn sync.Mutex - kv, err := js.KeyValue(ctx, headsBucket) + kv, err := essentialBucket(ctx, js, headsBucket, "keep the heads it wrote on the forge stream") if err != nil { - return fmt.Errorf("the bucket %s: %w", headsBucket, err) + return err } heads := kvHeads{kv} // Everything written on FORGE, an event's and a sync's, keeps the head in the bucket. rec := &recorder{pub: js, heads: heads, logger: logger} said := func(ctx context.Context) error { return syncOpen(ctx, call, rec, heads, &turn, logger) } // What the gate asks of the forge (ADR 0437 §8): its owed acts and its merges, each performed once per revision. - performedKV, err := js.KeyValue(ctx, performedBucket) + performedKV, err := essentialBucket(ctx, js, performedBucket, "perform an act of the gate only once") if err != nil { - return fmt.Errorf("the bucket %s: %w", performedBucket, err) + return err } perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger} watching := make(chan error, 3) @@ -207,16 +207,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro bucket string react func(ctx context.Context, key string, rev uint64, value []byte) }{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, perf.merges}} { - kv, err := js.KeyValue(ctx, w.bucket) - if errors.Is(err, jetstream.ErrBucketNotFound) { - // The gate's buckets come with the gate; the bridge does not wait for them (no timer), and is started again - // by its delivery when the gate arrives. - logger.Printf("the bucket %s does not exist yet: nothing of it is performed until the bridge starts again", w.bucket) + // The gate's buckets come with the gate and its grant; the bridge does not wait for them (no timer), and is started + // again by its delivery when the gate arrives. A bucket refused or not answered is the same (issue 1945). + kv := optionalBucket(ctx, js, w.bucket, "nothing of it is performed until the bridge starts again", logger.Printf) + if kv == nil { continue } - if err != nil { - return fmt.Errorf("the bucket %s: %w", w.bucket, err) - } go func() { watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) { hctx, cancel := context.WithTimeout(ctx, syncDeadline) @@ -231,7 +227,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro go onReconnect(ctx, again, said, logger) cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module) if err != nil { - return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, eventsStream, err) + // Essential: the forge's events are what the bridge carries. + return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the bridge hears no forge event without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, eventsStream, err) } cc, err := cons.Consume(func(m jetstream.Msg) { hear(&turn, func() { diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go index c4ab010..0f9cc32 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "log" "slices" "strings" @@ -66,6 +67,16 @@ func body(m map[string]any) []byte { } func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit part; the bus is the integration test) + t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) { + refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x") + var said []string + if kv := optionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") { + t.Errorf("an optional bucket refused: %v, said %v", kv, said) + } + if _, err := essentialBucket(context.Background(), refusingBus{refusal}, "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") { + t.Errorf("an essential bucket refused: %v, want its grant named", err) + } + }) var logs bytes.Buffer m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{} handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0)) @@ -230,3 +241,8 @@ func someFiles(_, _, path, _ string) ([]byte, error) { } return nil, bridge.ErrNotFound } + +// refusingBus refuses every bucket with err (issue 1945). +type refusingBus struct{ err error } + +func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/optional.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/optional.go new file mode 100644 index 0000000..68df917 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/optional.go @@ -0,0 +1,46 @@ +package main + +import ( + "context" + "fmt" + + "github.com/nats-io/nats.go/jetstream" +) + +// **A part does not stop for a bucket it can work without** (novox/hq issue 1945, met twice on 2026-10-25: the builder of +// mesh-delivery #48 exited on a refused read of the gate's source, and its new reasks bucket would have done the same). A +// grant reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted +// yet: exiting for it stops the part until a person's hand. A bucket the bridge cannot do its job at all without still stops +// the start, naming the grant to look at; every other is said and left. (The same two functions are in each delivery part; +// they belong in mesh-sdk.) + +// bucketOpener is the part of the bus a start opens buckets with. +type bucketOpener interface { + KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error) +} + +// essentialBucket opens a bucket the bridge cannot work without; its error names the bucket and the grant. +func essentialBucket(ctx context.Context, js bucketOpener, name, does string) (jetstream.KeyValue, error) { + kv, err := js.KeyValue(ctx, name) + if err != nil { + return nil, essentialErr(name, does, err) + } + return kv, nil +} + +// essentialErr is the error of a bucket the part cannot work without: it names the bucket and the grant. +func essentialErr(name, does string, err error) error { + return fmt.Errorf("the bucket %s cannot be read (%v): the bridge cannot %s without it, so it stops; where the bus "+ + "refused it or did not answer, this module's grant of %s (its manifest's reads or state, issued with its credential) "+ + "has not reached the bus yet", name, err, does, name) +} + +// optionalBucket opens a bucket the bridge works without: any error is said with what is not heard, and nil answered. +func optionalBucket(ctx context.Context, js bucketOpener, name, without string, logf func(string, ...any)) jetstream.KeyValue { + kv, err := js.KeyValue(ctx, name) + if err != nil { + logf("the bucket %s cannot be read (%v): %s", name, err, without) + return nil + } + return kv +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index fbf6ff3..0e32028 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -240,29 +240,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - bucket := func(name string) (jetstream.KeyValue, error) { - kv, err := js.KeyValue(ctx, name) - if err != nil { - return nil, fmt.Errorf("the bucket %s: %w", name, err) - } - return kv, nil - } - candidates, err := bucket(delivery.CandidatesBucket) + candidates, err := essentialBucket(ctx, js, delivery.CandidatesBucket, "write a candidate's fingerprints") if err != nil { return err } - modules, err := bucket(delivery.ModulesBucket) + modules, err := essentialBucket(ctx, js, delivery.ModulesBucket, "write a module's fingerprint") if err != nil { return err } - manifests, err := bucket(delivery.ManifestsBucket) - if err != nil { - return err - } - builds, err := bucket(delivery.BuildsBucket) + manifests, err := essentialBucket(ctx, js, delivery.ManifestsBucket, "keep a module's manifest") if err != nil { return err } + // The controller's build records only name the directory of a module whose key names none (fillPaths): without them + // such a module waits for its path, every other is indexed (issue 1945). + builds := optionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf) known := index.NewBuilds() // The controller's build records name the directory of a module whose key names none (fillPaths); no build of any // module moves a fingerprint since no fingerprint takes the builder (ADR 0530 §1). @@ -273,12 +265,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro known.Delete(e.Key()) } } - w, err := settle(ctx, builds, each) - if err != nil { - return err - } errs := make(chan error, 1) - go func() { errs <- follow(ctx, w, each) }() + if builds != nil { + if w, err := settle(ctx, builds, each); err != nil { + logger.Printf("the bucket %s cannot be watched (%v): a module whose key names no directory is not given one until the indexer starts again", delivery.BuildsBucket, err) + } else { + go func() { errs <- follow(ctx, w, each) }() + } + } // The indexer asks the forge for the objects of a manifest's named paths as its own account (ADR 0499 clause 3). account, err := process.ReadAccount(path) @@ -296,22 +290,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does // not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate. sourceErrs := make(chan error, 1) - switch source, err := js.KeyValue(ctx, delivery.SourceBucket); { - case errors.Is(err, jetstream.ErrBucketNotFound): - logger.Printf("the bucket %s does not exist yet: no source revision is read until the indexer starts again", delivery.SourceBucket) - case err != nil: - return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err) - default: + if source := optionalBucket(ctx, js, delivery.SourceBucket, "no source revision is read until the indexer starts again", logger.Printf); source != nil { each := func(e jetstream.KeyValueEntry) { if opOf(e) { applySource(ctx, ix, e.Key(), e.Revision(), e.Value(), logger) } } - sw, err := settle(ctx, source, each) - if err != nil { - return err + if sw, err := settle(ctx, source, each); err != nil { + logger.Printf("the bucket %s cannot be watched (%v): no source revision is read until the indexer starts again", delivery.SourceBucket, err) + } else { + go func() { sourceErrs <- follow(ctx, sw, each) }() } - go func() { sourceErrs <- follow(ctx, sw, each) }() } // Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's // assignments are settled first, so the toolchains settle through the gate already knowing them. @@ -381,7 +370,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } cons, err := js.Consumer(ctx, forgeStream, cred.Node+"_"+cred.Module) if err != nil { - return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, forgeStream, err) + // Essential: the forge's merges are what the indexer indexes. + return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the indexer hears no merge without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, forgeStream, err) } cc, err := cons.Consume(func(m jetstream.Msg) { handle(ctx, m, ix, logger) }, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) })) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/optional.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/optional.go new file mode 100644 index 0000000..1f1fb5c --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/optional.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + "fmt" + + "github.com/nats-io/nats.go/jetstream" +) + +// **A part does not stop for a bucket it can work without** (novox/hq issue 1945, met twice on 2026-10-25: the builder of +// mesh-delivery #48 exited on a refused read of the gate's source, and its new reasks bucket would have done the same). A +// grant reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted +// yet: exiting for it stops the part until a person's hand. A bucket the indexer cannot do its job at all without still stops +// the start, naming the grant to look at; every other is said and left. (The same two functions are in each delivery part; +// they belong in mesh-sdk.) + +// essentialBucket opens a bucket the indexer cannot work without; its error names the bucket and the grant. +func essentialBucket(ctx context.Context, js bucketOpener, name, does string) (jetstream.KeyValue, error) { + kv, err := js.KeyValue(ctx, name) + if err != nil { + return nil, essentialErr(name, does, err) + } + return kv, nil +} + +// essentialErr is the error of a bucket the part cannot work without: it names the bucket and the grant. +func essentialErr(name, does string, err error) error { + return fmt.Errorf("the bucket %s cannot be read (%v): the indexer cannot %s without it, so it stops; where the bus "+ + "refused it or did not answer, this module's grant of %s (its manifest's reads or state, issued with its credential) "+ + "has not reached the bus yet", name, err, does, name) +} + +// optionalBucket opens a bucket the indexer works without: any error is said with what is not heard, and nil answered. +func optionalBucket(ctx context.Context, js bucketOpener, name, without string, logf func(string, ...any)) jetstream.KeyValue { + kv, err := js.KeyValue(ctx, name) + if err != nil { + logf("the bucket %s cannot be read (%v): %s", name, err, without) + return nil + } + return kv +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go index c42221c..748182b 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go @@ -3,7 +3,6 @@ package main import ( "context" "errors" - "fmt" "log" "strings" "sync" @@ -55,11 +54,14 @@ func (w *previewWatch) ensure() error { w.logger.Printf("the bucket %s does not exist yet: it is watched when the checker says it wrote a preview", delivery.PreviewsBucket) return nil case err != nil: - return fmt.Errorf("the bucket %s: %w", delivery.PreviewsBucket, err) + // Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the checker's next preview. + w.logger.Printf("the bucket %s cannot be read (%v): it is watched when the checker says it wrote a preview", delivery.PreviewsBucket, err) + return nil } watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutPreview(e.Key(), e.Value(), !opOf(e)) }) if err != nil { - return err + w.logger.Printf("the bucket %s cannot be watched (%v): it is watched when the checker says it wrote a preview", delivery.PreviewsBucket, err) + return nil } w.watched = true go func() { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing.go index 3d6decf..9a06cec 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing.go @@ -193,7 +193,9 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) } w, err := settle(ctx, kv, each) if err != nil { - return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err) + // Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read. + logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err) + return errs, nil } if gate.Settled() { changed(ctx) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains.go index 30892ae..3e4f0cf 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains.go @@ -3,7 +3,6 @@ package main import ( "context" "errors" - "fmt" "log" "maps" "slices" @@ -73,11 +72,14 @@ func (w *toolchainWatch) ensure(module string) error { w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module) return nil case err != nil: - return fmt.Errorf("the bucket %s: %w", name, err) + // Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report. + w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module) + return nil } watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.sink.put(module, e.Key(), e.Value(), !opOf(e)) }) if err != nil { - return err + w.logger.Printf("the bucket %s cannot be watched (%v): %s's toolchains are not heard until its next report", name, err, module) + return nil } w.watched[module] = true go func() { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains_test.go index e26bf76..7427efb 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "errors" "log" "strings" "sync" @@ -21,6 +22,7 @@ type laterBuckets struct { mu sync.Mutex appears map[string]jetstream.KeyValue // buckets that exist now asked map[string]int + refuse error // answered for a bucket that does not exist, when set: a bus that refuses it (issue 1945) } func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) { @@ -30,6 +32,9 @@ func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyVa if kv, ok := l.appears[name]; ok { return kv, nil } + if l.refuse != nil { + return nil, l.refuse + } return nil, jetstream.ErrBucketNotFound } @@ -100,6 +105,12 @@ func TestABuildModulesBucketThatAppearsLaterIsWatchedWhenItsEventArrives(t *test recompute: func(context.Context) error { mu.Lock(); recomputes++; mu.Unlock(); return nil }, } var logs bytes.Buffer + // 1945: a bus that refuses both buckets, or does not answer, leaves them unwatched, said, and the indexer running. + refusing := &laterBuckets{appears: map[string]jetstream.KeyValue{}, asked: map[string]int{}, refuse: errors.New("context deadline exceeded")} + if err := newToolchainWatch(ctx, refusing, sink, log.New(&logs, "", 0)).start(); err != nil || !strings.Contains(logs.String(), "cannot be read") { + t.Fatalf("a refused toolchains bucket: start %v, log %q; want it said and the start going on", err, logs.String()) + } + logs.Reset() w := newToolchainWatch(ctx, buckets, sink, log.New(&logs, "", 0)) if err := w.start(); err != nil { // neither bucket exists: nothing is watched, nothing is written, nobody waits t.Fatal(err) diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go index f9b9142..8ed2b7b 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go @@ -119,12 +119,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return nil } if err != nil { - return fmt.Errorf("the bucket %s: %w", delivery.AsksBucket, err) + // Essential: the gate's asks are what the teller tells (issue 1945: the error names the grant). + return essentialErr(delivery.AsksBucket, "tell any of the gate's asks", err) } + // The answers it writes and what it asked: its own work, so each stops the start, naming its grant. var kvs [2]jetstream.KeyValue for i, name := range []string{delivery.AnswersBucket, askedBucket} { - if kvs[i], err = js.KeyValue(ctx, name); err != nil { - return fmt.Errorf("the bucket %s: %w", name, err) + if kvs[i], err = essentialBucket(ctx, js, name, "write an answer or remember what it asked"); err != nil { + return err } } state := kvStore{ctx: ctx, kv: kvs[1]} @@ -140,7 +142,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The view's answers to the assigner's asks (issue 1297 B10): both buckets are the controller's to create, and a bucket not // there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer). failed := make(chan error, 2) - assignerKV, viewKV := optionalBucket(ctx, js, delivery.AssignerAsksBucket, logger), optionalBucket(ctx, js, delivery.ViewAnswersBucket, logger) + assignerKV, viewKV := optionalBucket(ctx, js, delivery.AssignerAsksBucket, teller, logger.Printf), optionalBucket(ctx, js, delivery.ViewAnswersBucket, teller, logger.Printf) // The assigner's asks are raised on the operator channel whether or not the view's answers can be read (issue 1906). if viewKV != nil { t.View = kvGate{ctx: ctx, kv: viewKV} @@ -172,15 +174,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } } -// optionalBucket is a bucket the teller reads when it exists, and nil, said once, when it does not. -func optionalBucket(ctx context.Context, js jetstream.JetStream, name string, logger *log.Logger) jetstream.KeyValue { - kv, err := js.KeyValue(ctx, name) - if err != nil { - logger.Printf("the bucket %s cannot be read (%v): the asks and answers it holds are not carried until the teller is started again with it", name, err) - return nil - } - return kv -} +// teller is what an optional bucket of the teller leaves out. +const teller = "the asks and answers it holds are not carried until the teller is started again with it" // askedBucket is the teller's state bucket asked, `_`: what it asked of the router and which condition it // raised (manifest state entry asked, novox/hq ADR 0437 §4). diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/optional.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/optional.go new file mode 100644 index 0000000..9f8f3bc --- /dev/null +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/optional.go @@ -0,0 +1,46 @@ +package main + +import ( + "context" + "fmt" + + "github.com/nats-io/nats.go/jetstream" +) + +// **A part does not stop for a bucket it can work without** (novox/hq issue 1945, met twice on 2026-10-25: the builder of +// mesh-delivery #48 exited on a refused read of the gate's source, and its new reasks bucket would have done the same). A +// grant reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted +// yet: exiting for it stops the part until a person's hand. A bucket the teller cannot do its job at all without still stops +// the start, naming the grant to look at; every other is said and left. (The same two functions are in each delivery part; +// they belong in mesh-sdk.) + +// bucketOpener is the part of the bus a start opens buckets with. +type bucketOpener interface { + KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error) +} + +// essentialBucket opens a bucket the teller cannot work without; its error names the bucket and the grant. +func essentialBucket(ctx context.Context, js bucketOpener, name, does string) (jetstream.KeyValue, error) { + kv, err := js.KeyValue(ctx, name) + if err != nil { + return nil, essentialErr(name, does, err) + } + return kv, nil +} + +// essentialErr is the error of a bucket the part cannot work without: it names the bucket and the grant. +func essentialErr(name, does string, err error) error { + return fmt.Errorf("the bucket %s cannot be read (%v): the teller cannot %s without it, so it stops; where the bus "+ + "refused it or did not answer, this module's grant of %s (its manifest's reads or state, issued with its credential) "+ + "has not reached the bus yet", name, err, does, name) +} + +// optionalBucket opens a bucket the teller works without: any error is said with what is not heard, and nil answered. +func optionalBucket(ctx context.Context, js bucketOpener, name, without string, logf func(string, ...any)) jetstream.KeyValue { + kv, err := js.KeyValue(ctx, name) + if err != nil { + logf("the bucket %s cannot be read (%v): %s", name, err, without) + return nil + } + return kv +} diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go index 8986842..ce2cf62 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go @@ -1,8 +1,11 @@ package main import ( + "context" "encoding/json" + "errors" "fmt" + "github.com/nats-io/nats.go/jetstream" "strings" "testing" "time" @@ -236,6 +239,16 @@ func (b *bed) askedOf(subject string) []published { } func TestAnAsksKeyBecomesOneRouterAskAndDeletingItCancelsIt(t *testing.T) { // 1343/AC1 + t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) { + refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x") + var said []string + if kv := optionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") { + t.Errorf("an optional bucket refused: %v, said %v", kv, said) + } + if _, err := essentialBucket(context.Background(), refusingBus{refusal}, "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") { + t.Errorf("an essential bucket refused: %v, want its grant named", err) + } + }) b := newBed(t) key, value, ga := gateAsk(t, twoHeads(), twoHeads()) b.gate[key] = value @@ -518,3 +531,8 @@ func TestAnAnswerToAnEarlierAskingDoesNotBlockTheNextAsking(t *testing.T) { // R t.Errorf("writes %d asks %d dropped %v", b.answers.writes, len(b.askedOf(asks.AskSubject(delivery.TellerModule))), b.dropped) } } + +// refusingBus refuses every bucket with err (issue 1945). +type refusingBus struct{ err error } + +func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err } diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index f619da4..e59d27f 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -222,23 +222,23 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - bucket := func(name string) (jetstream.KeyValue, error) { - kv, err := js.KeyValue(ctx, name) - if err != nil { - return nil, fmt.Errorf("the bucket %s: %w", name, err) - } - return kv, nil - } + // Its own presence, pulses and stuck rows are what it keeps: each stops the start, naming its grant. The assigner's wants + // and the node-engine's haves only make the pulses of a move: without one the watchman watches presence and the build + // seats' steps, and says that moves are not watched (issue 1945). var kvs struct{ presence, pulse, stuck, wants, haves jetstream.KeyValue } for _, b := range []struct { name string into *jetstream.KeyValue - }{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck}, - {delivery.AssignerWantsBucket, &kvs.wants}, {delivery.HaveBucket, &kvs.haves}} { - if *b.into, err = bucket(b.name); err != nil { + }{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck}} { + if *b.into, err = essentialBucket(ctx, js, b.name, "say who is online or what stopped moving"); err != nil { return err } } + kvs.wants = optionalBucket(ctx, js, delivery.AssignerWantsBucket, "a want makes no pulse, so a move that stops is not seen, until the watchman starts again", logger.Printf) + kvs.haves = optionalBucket(ctx, js, delivery.HaveBucket, "a want makes no pulse, so a move that stops is not seen, until the watchman starts again", logger.Printf) + if kvs.wants == nil || kvs.haves == nil { + kvs.wants, kvs.haves = nil, nil // a want is a pulse only against the have that ends it + } w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck}, time.Now) w.Logf(logger.Printf) @@ -283,13 +283,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } // What the node agents run, then the assigner's wants (issue 1906, ADR 0539 §19): a want is a pulse owed by its node's agent // until the have names its fingerprint, so the haves are read first and a want already applied makes no pulse. - haveSeen, err := settle(ctx, kvs.haves, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }) - if err != nil { - return err - } - wantSeen, err := settle(ctx, kvs.wants, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) }) - if err != nil { - return err + var haveSeen, wantSeen jetstream.KeyWatcher + if kvs.haves != nil { + if haveSeen, err = settle(ctx, kvs.haves, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }); err != nil { + logger.Printf("the bucket %s cannot be watched (%v): moves are not watched until the watchman starts again", delivery.HaveBucket, err) + } else if wantSeen, err = settle(ctx, kvs.wants, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) }); err != nil { + logger.Printf("the bucket %s cannot be watched (%v): moves are not watched until the watchman starts again", delivery.AssignerWantsBucket, err) + haveSeen.Stop() + haveSeen = nil + } } // The build seats' steps: a step key `.` written is the ask taken (issue 1887 A8b). A seat whose build module // keeps no steps bucket (yet) is said and left; the asks of its stream are mirrored all the same. @@ -304,7 +306,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro lang := lang sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { stepTaken(w, lang, e) }) if err != nil { - return err + logger.Printf("the steps of %s cannot be watched (%v): its asks are mirrored and never seen taken", module, err) + continue } taken[lang] = sw } @@ -348,12 +351,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro go func() { errs <- follow(ctx, pulses, func(e jetstream.KeyValueEntry) { w.Pulse(e.Key(), opOf(e), e.Created()) }) }() - go func() { - errs <- follow(ctx, haveSeen, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }) - }() - go func() { - errs <- follow(ctx, wantSeen, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) }) - }() + if haveSeen != nil && wantSeen != nil { + go func() { + errs <- follow(ctx, haveSeen, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }) + }() + go func() { + errs <- follow(ctx, wantSeen, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) }) + }() + } for lang, sw := range taken { lang, sw := lang, sw go func() { errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { stepTaken(w, lang, e) }) }() diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go index 6a171c2..bee4b2e 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go @@ -2,11 +2,15 @@ package main import ( "context" + "errors" + "fmt" "io" "os" "path/filepath" "strings" "testing" + + "github.com/nats-io/nats.go/jetstream" ) func TestCredentialIsTheBrokerFile(t *testing.T) { // 1135/AC6 @@ -27,8 +31,23 @@ func TestCredentialIsTheBrokerFile(t *testing.T) { // 1135/AC6 } func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6 + t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) { + refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x") + var said []string + if kv := optionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") { + t.Errorf("an optional bucket refused: %v, said %v", kv, said) + } + if _, err := essentialBucket(context.Background(), refusingBus{refusal}, "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") { + t.Errorf("an essential bucket refused: %v, want its grant named", err) + } + }) err := run(context.Background(), func(string) string { return "" }, io.Discard) if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") { t.Fatalf("run = %v, want a refusal naming MESH_BROKER_FILE", err) } } + +// refusingBus refuses every bucket with err (issue 1945). +type refusingBus struct{ err error } + +func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err } diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/optional.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/optional.go new file mode 100644 index 0000000..4d3e2dc --- /dev/null +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/optional.go @@ -0,0 +1,46 @@ +package main + +import ( + "context" + "fmt" + + "github.com/nats-io/nats.go/jetstream" +) + +// **A part does not stop for a bucket it can work without** (novox/hq issue 1945, met twice on 2026-10-25: the builder of +// mesh-delivery #48 exited on a refused read of the gate's source, and its new reasks bucket would have done the same). A +// grant reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted +// yet: exiting for it stops the part until a person's hand. A bucket the watchman cannot do its job at all without still stops +// the start, naming the grant to look at; every other is said and left. (The same two functions are in each delivery part; +// they belong in mesh-sdk.) + +// bucketOpener is the part of the bus a start opens buckets with. +type bucketOpener interface { + KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error) +} + +// essentialBucket opens a bucket the watchman cannot work without; its error names the bucket and the grant. +func essentialBucket(ctx context.Context, js bucketOpener, name, does string) (jetstream.KeyValue, error) { + kv, err := js.KeyValue(ctx, name) + if err != nil { + return nil, essentialErr(name, does, err) + } + return kv, nil +} + +// essentialErr is the error of a bucket the part cannot work without: it names the bucket and the grant. +func essentialErr(name, does string, err error) error { + return fmt.Errorf("the bucket %s cannot be read (%v): the watchman cannot %s without it, so it stops; where the bus "+ + "refused it or did not answer, this module's grant of %s (its manifest's reads or state, issued with its credential) "+ + "has not reached the bus yet", name, err, does, name) +} + +// optionalBucket opens a bucket the watchman works without: any error is said with what is not heard, and nil answered. +func optionalBucket(ctx context.Context, js bucketOpener, name, without string, logf func(string, ...any)) jetstream.KeyValue { + kv, err := js.KeyValue(ctx, name) + if err != nil { + logf("the bucket %s cannot be read (%v): %s", name, err, without) + return nil + } + return kv +} From 4628ba26b41aae8e40d59f835ecdf13d85842926 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 25 Oct 2026 22:16:32 +0100 Subject: [PATCH 4/4] Delivery parts: take the essential and optional bucket opens from mesh-sdk (issue 1945) The open helpers were copied into seven parts; code shared between modules lives in mesh-sdk, so they are mesh-sdk process's now and the parts are pinned at fix/1945-optional-buckets (e0d1de04), moved to its main once it merges, one delivery group with mesh-sdk. --- .../cmd/mesh-delivery-assigner/main.go | 8 ++-- .../cmd/mesh-delivery-assigner/optional.go | 27 ----------- modules/mesh-delivery-assigner/go.mod | 2 +- modules/mesh-delivery-assigner/go.sum | 2 + .../cmd/mesh-delivery-builder/main.go | 19 ++++---- .../cmd/mesh-delivery-builder/optional.go | 28 +---------- .../mesh-delivery-builder/standing_test.go | 4 +- .../cmd/mesh-delivery-builder/toolchains.go | 7 ++- modules/mesh-delivery-builder/go.mod | 2 +- modules/mesh-delivery-builder/go.sum | 2 + .../cmd/mesh-delivery-checker/main.go | 12 ++--- .../cmd/mesh-delivery-checker/optional.go | 41 ----------------- .../mesh-delivery-checker/standing_test.go | 4 +- .../cmd/mesh-delivery-checker/toolchains.go | 5 +- modules/mesh-delivery-checker/go.mod | 2 +- modules/mesh-delivery-checker/go.sum | 2 + .../cmd/mesh-delivery-forge-bridge/main.go | 8 ++-- .../mesh-delivery-forge-bridge/main_test.go | 6 ++- .../mesh-delivery-forge-bridge/optional.go | 46 ------------------- modules/mesh-delivery-forge-bridge/go.mod | 2 +- modules/mesh-delivery-forge-bridge/go.sum | 2 + .../cmd/mesh-delivery-indexer/main.go | 14 +++--- .../cmd/mesh-delivery-indexer/optional.go | 41 ----------------- .../cmd/mesh-delivery-indexer/previews.go | 3 +- .../cmd/mesh-delivery-indexer/toolchains.go | 5 +- modules/mesh-delivery-indexer/go.mod | 2 +- modules/mesh-delivery-indexer/go.sum | 2 + .../cmd/mesh-delivery-teller/main.go | 8 ++-- .../cmd/mesh-delivery-teller/optional.go | 46 ------------------- .../cmd/mesh-delivery-teller/teller_test.go | 8 ++-- modules/mesh-delivery-teller/go.mod | 2 +- modules/mesh-delivery-teller/go.sum | 2 + .../cmd/mesh-delivery-view-tools/main.go | 3 +- .../cmd/mesh-delivery-watchman/main.go | 12 +++-- .../cmd/mesh-delivery-watchman/main_test.go | 6 ++- .../cmd/mesh-delivery-watchman/optional.go | 46 ------------------- modules/mesh-delivery-watchman/go.mod | 2 +- modules/mesh-delivery-watchman/go.sum | 2 + 38 files changed, 102 insertions(+), 333 deletions(-) delete mode 100644 modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/optional.go delete mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/optional.go delete mode 100644 modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/optional.go delete mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/optional.go delete mode 100644 modules/mesh-delivery-teller/cmd/mesh-delivery-teller/optional.go delete mode 100644 modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/optional.go diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 2f2512d..3ba0133 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -86,11 +86,11 @@ var keptStates = []announce.State{ // buckets is every bucket the assigner opened, by name. type buckets map[string]jetstream.KeyValue -// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named with its grant (issue 1945), since a pass -// without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it +// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named with its grant (issue 1945), +// since a pass without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it // does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants // without it and takes no put further until it starts again (ADR 0546). -func open(ctx context.Context, js bucketOpener, logf func(string, ...any)) (buckets, error) { +func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) { out := buckets{} for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) { kv, err := js.KeyValue(ctx, name) @@ -99,7 +99,7 @@ func open(ctx context.Context, js bucketOpener, logf func(string, ...any)) (buck continue } if err != nil { - return nil, essentialErr(name, "write a want from the whole mesh", err) + return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err) } out[name] = kv } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/optional.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/optional.go deleted file mode 100644 index 0267144..0000000 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/optional.go +++ /dev/null @@ -1,27 +0,0 @@ -package main - -import ( - "context" - "fmt" - - "github.com/nats-io/nats.go/jetstream" -) - -// **A part does not stop for a bucket it can work without** (novox/hq issue 1945, met twice on 2026-10-25: the builder of -// mesh-delivery #48 exited on a refused read of the gate's source, and its new reasks bucket would have done the same). A -// grant reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted -// yet: exiting for it stops the part until a person's hand. The assigner works without none of its buckets but the Go build -// module's steps, so the others stop the start, naming the grant to look at. (The same two functions are in each delivery part; -// they belong in mesh-sdk.) - -// bucketOpener is the part of the bus a start opens buckets with. -type bucketOpener interface { - KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error) -} - -// essentialErr is the error of a bucket the part cannot work without: it names the bucket and the grant. -func essentialErr(name, does string, err error) error { - return fmt.Errorf("the bucket %s cannot be read (%v): the assigner cannot %s without it, so it stops; where the bus "+ - "refused it or did not answer, this module's grant of %s (its manifest's reads or state, issued with its credential) "+ - "has not reached the bus yet", name, err, does, name) -} diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index 893c786..df8760f 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index 2d6d7be..8492faa 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -6,6 +6,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv4 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index d6b2e9f..1726dc5 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -13,7 +13,6 @@ import ( "encoding/json" "errors" "fmt" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "io" "log" "maps" @@ -25,6 +24,8 @@ import ( "syscall" "time" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -124,20 +125,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - builds, err := essentialBucket(ctx, js, delivery.BuilderBuildsBucket, "claim or write a build key") + builds, err := process.EssentialBucket(ctx, js, "builder", delivery.BuilderBuildsBucket, "claim or write a build key") if err != nil { return err } - modules, err := essentialBucket(ctx, js, delivery.ModulesBucket, "know any fingerprint to build") + modules, err := process.EssentialBucket(ctx, js, "builder", delivery.ModulesBucket, "know any fingerprint to build") if err != nil { return err } - manifests, err := essentialBucket(ctx, js, delivery.ManifestsBucket, "read what a module builds") + manifests, err := process.EssentialBucket(ctx, js, "builder", delivery.ManifestsBucket, "read what a module builds") if err != nil { return err } // The candidates' fingerprints are built too (issue 1684): a builder that cannot read them builds the trunk's. - candidates := optionalBucket(ctx, js, delivery.CandidatesBucket, "no candidate's fingerprint is built until the builder starts again; the trunk's are", logger.Printf) + candidates := process.OptionalBucket(ctx, js, delivery.CandidatesBucket, "no candidate's fingerprint is built until the builder starts again; the trunk's are", logger.Printf) bound := defaultStepBound if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" { if bound, err = time.ParseDuration(v); err != nil { @@ -188,7 +189,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // controller delivers from this builder's keys only while an instance says it writes references. A bucket not there // yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent // is asked since R4g, ADR 0524 §7). - switch kv := optionalBucket(ctx, js, instancesBucket, "the controller is not told that this builder writes references until it starts again", logger.Printf); { + switch kv := process.OptionalBucket(ctx, js, instancesBucket, "the controller is not told that this builder writes references until it starts again", logger.Printf); { case kv == nil: default: says := newSayer(ctx, kv, cred.Node, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, build.ImageBuildModule}, logger.Printf) @@ -272,17 +273,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if candidates != nil { cw, err := settle(ctx, candidates, eachCandidate) if err != nil { - logger.Printf("the bucket %s cannot be watched (%v): no candidate's fingerprint is built until the builder starts again", delivery.CandidatesBucket, err) + process.NotWatched(logger.Printf, delivery.CandidatesBucket, err, "no candidate's fingerprint is built until the builder starts again") } else { go func() { errs <- follow(ctx, cw, eachCandidate) }() } } // A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as // every other input. The builder asks again by itself without it. - if kv := optionalBucket(ctx, js, build.ReAsksBucket, "builder_ask_again is not heard until the builder starts again; an ask that can no longer be answered is still asked again by itself", logger.Printf); kv != nil { + if kv := process.OptionalBucket(ctx, js, build.ReAsksBucket, "builder_ask_again is not heard until the builder starts again; an ask that can no longer be answered is still asked again by itself", logger.Printf); kv != nil { eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) } if rw, err := settle(ctx, kv, eachReAsk); err != nil { - logger.Printf("the bucket %s cannot be watched (%v): builder_ask_again is not heard until the builder starts again", build.ReAsksBucket, err) + process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again") } else { go func() { errs <- follow(ctx, rw, eachReAsk) }() } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go index bd2542e..02afe10 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go @@ -10,39 +10,13 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" ) -// **A part does not stop for a bucket it can work without** (novox/hq issue 1945, met twice on 2026-10-25: the builder of +// **A part does not stop for a bucket it can work without** (mesh-sdk process.EssentialBucket, OptionalBucket) (novox/hq issue 1945, met twice on 2026-10-25: the builder of // mesh-delivery #48 exited on a refused read of the gate's source, and its reasks bucket would have done the same). A grant // reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted yet: // exiting for it stops every build until a person's hand. A bucket the builder cannot build at all without (its builds, // the indexer's modules and manifests) still stops the start, naming the grant to look at; every other is said and left, // and opened again at the next event that needs it where there is one. -// essentialBucket opens a bucket the builder cannot work without; its error names the bucket and the grant. -func essentialBucket(ctx context.Context, js bucketOpener, name, does string) (jetstream.KeyValue, error) { - kv, err := js.KeyValue(ctx, name) - if err != nil { - return nil, essentialErr(name, does, err) - } - return kv, nil -} - -// essentialErr is the error of a bucket the part cannot work without: it names the bucket and the grant. -func essentialErr(name, does string, err error) error { - return fmt.Errorf("the bucket %s cannot be read (%v): the builder cannot %s without it, so it stops; where the bus "+ - "refused it or did not answer, this module's grant of %s (its manifest's reads or state, issued with its credential) "+ - "has not reached the bus yet", name, err, does, name) -} - -// optionalBucket opens a bucket the builder works without: any error is said with what is not heard, and nil answered. -func optionalBucket(ctx context.Context, js bucketOpener, name, without string, logf func(string, ...any)) jetstream.KeyValue { - kv, err := js.KeyValue(ctx, name) - if err != nil { - logf("the bucket %s cannot be read (%v): %s", name, err, without) - return nil - } - return kv -} - // trunkSource is the gate's source history the candidates' previews are merged onto (sourceHeads), opened at start and, // while it cannot be, again at the next candidate that asks a trunk head: a builder without it builds the trunk's keys, // and a candidate's preview waits, saying why. diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go index ae19091..4a0e7ca 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go @@ -12,6 +12,8 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -111,7 +113,7 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) { if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") { t.Errorf("the refusals were not said: %q", logs.String()) } - if _, err := essentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "mesh-delivery-indexer_modules", "work"); err == nil || + if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil || !strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") { t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go index f574c2d..58ae514 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go @@ -3,11 +3,14 @@ package main import ( "context" "errors" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" + "fmt" "log" "strings" "sync" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -67,7 +70,7 @@ func (w *toolchainWatch) ensure(module string) error { } watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) }) if err != nil { - w.logger.Printf("the bucket %s cannot be watched (%v): %s's toolchains are not heard until its next report", name, err, module) + process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module)) return nil } w.watched[module] = true diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index 615cce0..fb89b4d 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index d648ca0..2dfa4fd 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 7b97f0d..c46ec4d 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -157,7 +157,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro buckets := map[string]jetstream.KeyValue{} for _, name := range []string{delivery.CheckerVerdictsBucket, delivery.CheckerKeptBucket, delivery.CheckerOwedBucket, delivery.PreviewsBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.ManifestsBucket, check.TriesBucket} { - if buckets[name], err = essentialBucket(ctx, js, name, "judge or keep a check"); err != nil { + if buckets[name], err = process.EssentialBucket(ctx, js, "checker", name, "judge or keep a check"); err != nil { return err } } @@ -207,7 +207,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line // says nothing of the builds, until the checker starts again. var builds check.BuildStates - if kv := optionalBucket(ctx, js, delivery.BuilderBuildsBucket, "the line says nothing of the builds until the checker starts again", logger.Printf); kv != nil { + if kv := process.OptionalBucket(ctx, js, delivery.BuilderBuildsBucket, "the line says nothing of the builds until the checker starts again", logger.Printf); kv != nil { builds = buildReader{kvStore{kv}} } // The assigner's compositions (ADR 0528 clause 5, issue 1297): their bucket is looked up at each check until it is @@ -280,9 +280,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // the assigner's own presence says whether a composition is being made (compositionWatch, ADR 0537 §4). A check is no // longer held for a node: its asks wait in the build seats' streams, which redeliver a lost node's ask (ADR 0525 §6, // issue 1912), so the node-engine's have is not watched here. - if kv := optionalBucket(ctx, js, delivery.PresenceBucket, "no node counts as online until the checker starts again", logger.Printf); kv != nil { + if kv := process.OptionalBucket(ctx, js, delivery.PresenceBucket, "no node counts as online until the checker starts again", logger.Printf); kv != nil { if pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) }); err != nil { - logger.Printf("the bucket %s cannot be watched (%v): no node counts as online until the checker starts again", delivery.PresenceBucket, err) + process.NotWatched(logger.Printf, delivery.PresenceBucket, err, "no node counts as online until the checker starts again") } else { go func() { errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) { @@ -295,14 +295,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } // The gate's source says the trunk head of each repository, which a pull request is merged onto. Where it is not there // yet no trunk is known, so nothing is checked. - if kv := optionalBucket(ctx, js, delivery.SourceBucket, "no trunk is known, so nothing is checked until the checker starts again", logger.Printf); kv != nil { + if kv := process.OptionalBucket(ctx, js, delivery.SourceBucket, "no trunk is known, so nothing is checked until the checker starts again", logger.Printf); kv != nil { sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { if e.Key() == delivery.SourceMainKey { trunks.Put(e.Value(), !opOf(e)) } }) if err != nil { - logger.Printf("the bucket %s cannot be watched (%v): no trunk is known, so nothing is checked until the checker starts again", delivery.SourceBucket, err) + process.NotWatched(logger.Printf, delivery.SourceBucket, err, "no trunk is known, so nothing is checked until the checker starts again") } else { go func() { errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/optional.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/optional.go deleted file mode 100644 index 466cdd0..0000000 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/optional.go +++ /dev/null @@ -1,41 +0,0 @@ -package main - -import ( - "context" - "fmt" - - "github.com/nats-io/nats.go/jetstream" -) - -// **A part does not stop for a bucket it can work without** (novox/hq issue 1945, met twice on 2026-10-25: the builder of -// mesh-delivery #48 exited on a refused read of the gate's source, and its new reasks bucket would have done the same). A -// grant reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted -// yet: exiting for it stops the part until a person's hand. A bucket the checker cannot do its job at all without still stops -// the start, naming the grant to look at; every other is said and left. (The same two functions are in each delivery part; -// they belong in mesh-sdk.) - -// essentialBucket opens a bucket the checker cannot work without; its error names the bucket and the grant. -func essentialBucket(ctx context.Context, js bucketOpener, name, does string) (jetstream.KeyValue, error) { - kv, err := js.KeyValue(ctx, name) - if err != nil { - return nil, essentialErr(name, does, err) - } - return kv, nil -} - -// essentialErr is the error of a bucket the part cannot work without: it names the bucket and the grant. -func essentialErr(name, does string, err error) error { - return fmt.Errorf("the bucket %s cannot be read (%v): the checker cannot %s without it, so it stops; where the bus "+ - "refused it or did not answer, this module's grant of %s (its manifest's reads or state, issued with its credential) "+ - "has not reached the bus yet", name, err, does, name) -} - -// optionalBucket opens a bucket the checker works without: any error is said with what is not heard, and nil answered. -func optionalBucket(ctx context.Context, js bucketOpener, name, without string, logf func(string, ...any)) jetstream.KeyValue { - kv, err := js.KeyValue(ctx, name) - if err != nil { - logf("the bucket %s cannot be read (%v): %s", name, err, without) - return nil - } - return kv -} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go index c759d3a..0715aea 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go @@ -12,6 +12,8 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -111,7 +113,7 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) { if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") { t.Errorf("the refusals were not said: %q", logs.String()) } - if _, err := essentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "mesh-delivery-indexer_modules", "work"); err == nil || + if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil || !strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") { t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err) } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go index 0fe2f41..99c3087 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go @@ -3,10 +3,13 @@ package main import ( "context" "errors" + "fmt" "log" "strings" "sync" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -68,7 +71,7 @@ func (w *toolchainWatch) ensure(module string) error { } watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) }) if err != nil { - w.logger.Printf("the bucket %s cannot be watched (%v): %s's toolchains are not heard until its next report", name, err, module) + process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module)) return nil } w.watched[module] = true diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index e753fa3..5c4d652 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-checker go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum index 744af4c..4e82aa0 100644 --- a/modules/mesh-delivery-checker/go.sum +++ b/modules/mesh-delivery-checker/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 h1:FAiFY2KBKEoZSH49OZTt7/zRBIQxyl5g/KhYrpggbN4= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index 383bedc..e069a6c 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -22,6 +22,8 @@ import ( "syscall" "time" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -188,7 +190,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro call := callerOn(conn, cred.User) // No forge event is heard while the open pull requests are read and said (ADR 0443 §5): events and syncs take turns. var turn sync.Mutex - kv, err := essentialBucket(ctx, js, headsBucket, "keep the heads it wrote on the forge stream") + kv, err := process.EssentialBucket(ctx, js, "bridge", headsBucket, "keep the heads it wrote on the forge stream") if err != nil { return err } @@ -197,7 +199,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro rec := &recorder{pub: js, heads: heads, logger: logger} said := func(ctx context.Context) error { return syncOpen(ctx, call, rec, heads, &turn, logger) } // What the gate asks of the forge (ADR 0437 §8): its owed acts and its merges, each performed once per revision. - performedKV, err := essentialBucket(ctx, js, performedBucket, "perform an act of the gate only once") + performedKV, err := process.EssentialBucket(ctx, js, "bridge", performedBucket, "perform an act of the gate only once") if err != nil { return err } @@ -209,7 +211,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, perf.merges}} { // The gate's buckets come with the gate and its grant; the bridge does not wait for them (no timer), and is started // again by its delivery when the gate arrives. A bucket refused or not answered is the same (issue 1945). - kv := optionalBucket(ctx, js, w.bucket, "nothing of it is performed until the bridge starts again", logger.Printf) + kv := process.OptionalBucket(ctx, js, w.bucket, "nothing of it is performed until the bridge starts again", logger.Printf) if kv == nil { continue } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go index 0f9cc32..9a16a64 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go @@ -13,6 +13,8 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -70,10 +72,10 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) { refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x") var said []string - if kv := optionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") { + if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") { t.Errorf("an optional bucket refused: %v, said %v", kv, said) } - if _, err := essentialBucket(context.Background(), refusingBus{refusal}, "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") { + if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "bridge", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") { t.Errorf("an essential bucket refused: %v, want its grant named", err) } }) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/optional.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/optional.go deleted file mode 100644 index 68df917..0000000 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/optional.go +++ /dev/null @@ -1,46 +0,0 @@ -package main - -import ( - "context" - "fmt" - - "github.com/nats-io/nats.go/jetstream" -) - -// **A part does not stop for a bucket it can work without** (novox/hq issue 1945, met twice on 2026-10-25: the builder of -// mesh-delivery #48 exited on a refused read of the gate's source, and its new reasks bucket would have done the same). A -// grant reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted -// yet: exiting for it stops the part until a person's hand. A bucket the bridge cannot do its job at all without still stops -// the start, naming the grant to look at; every other is said and left. (The same two functions are in each delivery part; -// they belong in mesh-sdk.) - -// bucketOpener is the part of the bus a start opens buckets with. -type bucketOpener interface { - KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error) -} - -// essentialBucket opens a bucket the bridge cannot work without; its error names the bucket and the grant. -func essentialBucket(ctx context.Context, js bucketOpener, name, does string) (jetstream.KeyValue, error) { - kv, err := js.KeyValue(ctx, name) - if err != nil { - return nil, essentialErr(name, does, err) - } - return kv, nil -} - -// essentialErr is the error of a bucket the part cannot work without: it names the bucket and the grant. -func essentialErr(name, does string, err error) error { - return fmt.Errorf("the bucket %s cannot be read (%v): the bridge cannot %s without it, so it stops; where the bus "+ - "refused it or did not answer, this module's grant of %s (its manifest's reads or state, issued with its credential) "+ - "has not reached the bus yet", name, err, does, name) -} - -// optionalBucket opens a bucket the bridge works without: any error is said with what is not heard, and nil answered. -func optionalBucket(ctx context.Context, js bucketOpener, name, without string, logf func(string, ...any)) jetstream.KeyValue { - kv, err := js.KeyValue(ctx, name) - if err != nil { - logf("the bucket %s cannot be read (%v): %s", name, err, without) - return nil - } - return kv -} diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index 669a04d..6f902ec 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-forge-bridge/go.sum b/modules/mesh-delivery-forge-bridge/go.sum index db1ea18..bdc8b83 100644 --- a/modules/mesh-delivery-forge-bridge/go.sum +++ b/modules/mesh-delivery-forge-bridge/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 0e32028..a4a515f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -240,21 +240,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - candidates, err := essentialBucket(ctx, js, delivery.CandidatesBucket, "write a candidate's fingerprints") + candidates, err := process.EssentialBucket(ctx, js, "indexer", delivery.CandidatesBucket, "write a candidate's fingerprints") if err != nil { return err } - modules, err := essentialBucket(ctx, js, delivery.ModulesBucket, "write a module's fingerprint") + modules, err := process.EssentialBucket(ctx, js, "indexer", delivery.ModulesBucket, "write a module's fingerprint") if err != nil { return err } - manifests, err := essentialBucket(ctx, js, delivery.ManifestsBucket, "keep a module's manifest") + manifests, err := process.EssentialBucket(ctx, js, "indexer", delivery.ManifestsBucket, "keep a module's manifest") if err != nil { return err } // The controller's build records only name the directory of a module whose key names none (fillPaths): without them // such a module waits for its path, every other is indexed (issue 1945). - builds := optionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf) + builds := process.OptionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf) known := index.NewBuilds() // The controller's build records name the directory of a module whose key names none (fillPaths); no build of any // module moves a fingerprint since no fingerprint takes the builder (ADR 0530 §1). @@ -268,7 +268,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro errs := make(chan error, 1) if builds != nil { if w, err := settle(ctx, builds, each); err != nil { - logger.Printf("the bucket %s cannot be watched (%v): a module whose key names no directory is not given one until the indexer starts again", delivery.BuildsBucket, err) + process.NotWatched(logger.Printf, delivery.BuildsBucket, err, "a module whose key names no directory is not given one until the indexer starts again") } else { go func() { errs <- follow(ctx, w, each) }() } @@ -290,14 +290,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does // not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate. sourceErrs := make(chan error, 1) - if source := optionalBucket(ctx, js, delivery.SourceBucket, "no source revision is read until the indexer starts again", logger.Printf); source != nil { + if source := process.OptionalBucket(ctx, js, delivery.SourceBucket, "no source revision is read until the indexer starts again", logger.Printf); source != nil { each := func(e jetstream.KeyValueEntry) { if opOf(e) { applySource(ctx, ix, e.Key(), e.Revision(), e.Value(), logger) } } if sw, err := settle(ctx, source, each); err != nil { - logger.Printf("the bucket %s cannot be watched (%v): no source revision is read until the indexer starts again", delivery.SourceBucket, err) + process.NotWatched(logger.Printf, delivery.SourceBucket, err, "no source revision is read until the indexer starts again") } else { go func() { sourceErrs <- follow(ctx, sw, each) }() } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/optional.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/optional.go deleted file mode 100644 index 1f1fb5c..0000000 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/optional.go +++ /dev/null @@ -1,41 +0,0 @@ -package main - -import ( - "context" - "fmt" - - "github.com/nats-io/nats.go/jetstream" -) - -// **A part does not stop for a bucket it can work without** (novox/hq issue 1945, met twice on 2026-10-25: the builder of -// mesh-delivery #48 exited on a refused read of the gate's source, and its new reasks bucket would have done the same). A -// grant reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted -// yet: exiting for it stops the part until a person's hand. A bucket the indexer cannot do its job at all without still stops -// the start, naming the grant to look at; every other is said and left. (The same two functions are in each delivery part; -// they belong in mesh-sdk.) - -// essentialBucket opens a bucket the indexer cannot work without; its error names the bucket and the grant. -func essentialBucket(ctx context.Context, js bucketOpener, name, does string) (jetstream.KeyValue, error) { - kv, err := js.KeyValue(ctx, name) - if err != nil { - return nil, essentialErr(name, does, err) - } - return kv, nil -} - -// essentialErr is the error of a bucket the part cannot work without: it names the bucket and the grant. -func essentialErr(name, does string, err error) error { - return fmt.Errorf("the bucket %s cannot be read (%v): the indexer cannot %s without it, so it stops; where the bus "+ - "refused it or did not answer, this module's grant of %s (its manifest's reads or state, issued with its credential) "+ - "has not reached the bus yet", name, err, does, name) -} - -// optionalBucket opens a bucket the indexer works without: any error is said with what is not heard, and nil answered. -func optionalBucket(ctx context.Context, js bucketOpener, name, without string, logf func(string, ...any)) jetstream.KeyValue { - kv, err := js.KeyValue(ctx, name) - if err != nil { - logf("the bucket %s cannot be read (%v): %s", name, err, without) - return nil - } - return kv -} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go index 748182b..3b95a25 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "git.novox.be/novox/mesh-sdk/go/process" "log" "strings" "sync" @@ -60,7 +61,7 @@ func (w *previewWatch) ensure() error { } watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutPreview(e.Key(), e.Value(), !opOf(e)) }) if err != nil { - w.logger.Printf("the bucket %s cannot be watched (%v): it is watched when the checker says it wrote a preview", delivery.PreviewsBucket, err) + process.NotWatched(w.logger.Printf, delivery.PreviewsBucket, err, "it is watched when the checker says it wrote a preview") return nil } w.watched = true diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains.go index 3e4f0cf..92aafa4 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains.go @@ -3,12 +3,15 @@ package main import ( "context" "errors" + "fmt" "log" "maps" "slices" "strings" "sync" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -78,7 +81,7 @@ func (w *toolchainWatch) ensure(module string) error { } watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.sink.put(module, e.Key(), e.Value(), !opOf(e)) }) if err != nil { - w.logger.Printf("the bucket %s cannot be watched (%v): %s's toolchains are not heard until its next report", name, err, module) + process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module)) return nil } w.watched[module] = true diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 0aaff15..e8728ce 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 2a3e9f3..8e94e90 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go index 8ed2b7b..d73893d 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go @@ -22,6 +22,8 @@ import ( "strings" "syscall" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -120,12 +122,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } if err != nil { // Essential: the gate's asks are what the teller tells (issue 1945: the error names the grant). - return essentialErr(delivery.AsksBucket, "tell any of the gate's asks", err) + return process.EssentialErr("teller", delivery.AsksBucket, "tell any of the gate's asks", err) } // The answers it writes and what it asked: its own work, so each stops the start, naming its grant. var kvs [2]jetstream.KeyValue for i, name := range []string{delivery.AnswersBucket, askedBucket} { - if kvs[i], err = essentialBucket(ctx, js, name, "write an answer or remember what it asked"); err != nil { + if kvs[i], err = process.EssentialBucket(ctx, js, "teller", name, "write an answer or remember what it asked"); err != nil { return err } } @@ -142,7 +144,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The view's answers to the assigner's asks (issue 1297 B10): both buckets are the controller's to create, and a bucket not // there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer). failed := make(chan error, 2) - assignerKV, viewKV := optionalBucket(ctx, js, delivery.AssignerAsksBucket, teller, logger.Printf), optionalBucket(ctx, js, delivery.ViewAnswersBucket, teller, logger.Printf) + assignerKV, viewKV := process.OptionalBucket(ctx, js, delivery.AssignerAsksBucket, teller, logger.Printf), process.OptionalBucket(ctx, js, delivery.ViewAnswersBucket, teller, logger.Printf) // The assigner's asks are raised on the operator channel whether or not the view's answers can be read (issue 1906). if viewKV != nil { t.View = kvGate{ctx: ctx, kv: viewKV} diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/optional.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/optional.go deleted file mode 100644 index 9f8f3bc..0000000 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/optional.go +++ /dev/null @@ -1,46 +0,0 @@ -package main - -import ( - "context" - "fmt" - - "github.com/nats-io/nats.go/jetstream" -) - -// **A part does not stop for a bucket it can work without** (novox/hq issue 1945, met twice on 2026-10-25: the builder of -// mesh-delivery #48 exited on a refused read of the gate's source, and its new reasks bucket would have done the same). A -// grant reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted -// yet: exiting for it stops the part until a person's hand. A bucket the teller cannot do its job at all without still stops -// the start, naming the grant to look at; every other is said and left. (The same two functions are in each delivery part; -// they belong in mesh-sdk.) - -// bucketOpener is the part of the bus a start opens buckets with. -type bucketOpener interface { - KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error) -} - -// essentialBucket opens a bucket the teller cannot work without; its error names the bucket and the grant. -func essentialBucket(ctx context.Context, js bucketOpener, name, does string) (jetstream.KeyValue, error) { - kv, err := js.KeyValue(ctx, name) - if err != nil { - return nil, essentialErr(name, does, err) - } - return kv, nil -} - -// essentialErr is the error of a bucket the part cannot work without: it names the bucket and the grant. -func essentialErr(name, does string, err error) error { - return fmt.Errorf("the bucket %s cannot be read (%v): the teller cannot %s without it, so it stops; where the bus "+ - "refused it or did not answer, this module's grant of %s (its manifest's reads or state, issued with its credential) "+ - "has not reached the bus yet", name, err, does, name) -} - -// optionalBucket opens a bucket the teller works without: any error is said with what is not heard, and nil answered. -func optionalBucket(ctx context.Context, js bucketOpener, name, without string, logf func(string, ...any)) jetstream.KeyValue { - kv, err := js.KeyValue(ctx, name) - if err != nil { - logf("the bucket %s cannot be read (%v): %s", name, err, without) - return nil - } - return kv -} diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go index ce2cf62..ae05ae0 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go @@ -5,11 +5,13 @@ import ( "encoding/json" "errors" "fmt" - "github.com/nats-io/nats.go/jetstream" "strings" "testing" "time" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go/jetstream" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/asker" "git.novox.be/novox/mesh-sdk/go/asks" @@ -242,10 +244,10 @@ func TestAnAsksKeyBecomesOneRouterAskAndDeletingItCancelsIt(t *testing.T) { // 1 t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) { refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x") var said []string - if kv := optionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") { + if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") { t.Errorf("an optional bucket refused: %v, said %v", kv, said) } - if _, err := essentialBucket(context.Background(), refusingBus{refusal}, "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") { + if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "teller", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") { t.Errorf("an essential bucket refused: %v, want its grant named", err) } }) diff --git a/modules/mesh-delivery-teller/go.mod b/modules/mesh-delivery-teller/go.mod index db3470d..0937a16 100644 --- a/modules/mesh-delivery-teller/go.mod +++ b/modules/mesh-delivery-teller/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-teller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-teller/go.sum b/modules/mesh-delivery-teller/go.sum index 97c314d..3baa2a5 100644 --- a/modules/mesh-delivery-teller/go.sum +++ b/modules/mesh-delivery-teller/go.sum @@ -6,6 +6,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv4 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index 3ffe7d1..d0675db 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -9,8 +9,9 @@ import ( "os" "strings" - stdio "git.novox.be/novox/mesh-sdk/go" "mesh-delivery-view/internal/view" + + stdio "git.novox.be/novox/mesh-sdk/go" ) // Tools are the read tools over a modules state. diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index e59d27f..57994b4 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -22,6 +22,8 @@ import ( "syscall" "time" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -230,12 +232,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro name string into *jetstream.KeyValue }{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck}} { - if *b.into, err = essentialBucket(ctx, js, b.name, "say who is online or what stopped moving"); err != nil { + if *b.into, err = process.EssentialBucket(ctx, js, "watchman", b.name, "say who is online or what stopped moving"); err != nil { return err } } - kvs.wants = optionalBucket(ctx, js, delivery.AssignerWantsBucket, "a want makes no pulse, so a move that stops is not seen, until the watchman starts again", logger.Printf) - kvs.haves = optionalBucket(ctx, js, delivery.HaveBucket, "a want makes no pulse, so a move that stops is not seen, until the watchman starts again", logger.Printf) + kvs.wants = process.OptionalBucket(ctx, js, delivery.AssignerWantsBucket, "a want makes no pulse, so a move that stops is not seen, until the watchman starts again", logger.Printf) + kvs.haves = process.OptionalBucket(ctx, js, delivery.HaveBucket, "a want makes no pulse, so a move that stops is not seen, until the watchman starts again", logger.Printf) if kvs.wants == nil || kvs.haves == nil { kvs.wants, kvs.haves = nil, nil // a want is a pulse only against the have that ends it } @@ -286,9 +288,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro var haveSeen, wantSeen jetstream.KeyWatcher if kvs.haves != nil { if haveSeen, err = settle(ctx, kvs.haves, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }); err != nil { - logger.Printf("the bucket %s cannot be watched (%v): moves are not watched until the watchman starts again", delivery.HaveBucket, err) + process.NotWatched(logger.Printf, delivery.HaveBucket, err, "moves are not watched until the watchman starts again") } else if wantSeen, err = settle(ctx, kvs.wants, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) }); err != nil { - logger.Printf("the bucket %s cannot be watched (%v): moves are not watched until the watchman starts again", delivery.AssignerWantsBucket, err) + process.NotWatched(logger.Printf, delivery.AssignerWantsBucket, err, "moves are not watched until the watchman starts again") haveSeen.Stop() haveSeen = nil } diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go index bee4b2e..58fe0a7 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go @@ -10,6 +10,8 @@ import ( "strings" "testing" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go/jetstream" ) @@ -34,10 +36,10 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6 t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) { refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x") var said []string - if kv := optionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") { + if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") { t.Errorf("an optional bucket refused: %v, said %v", kv, said) } - if _, err := essentialBucket(context.Background(), refusingBus{refusal}, "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") { + if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "watchman", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") { t.Errorf("an essential bucket refused: %v, want its grant named", err) } }) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/optional.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/optional.go deleted file mode 100644 index 4d3e2dc..0000000 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/optional.go +++ /dev/null @@ -1,46 +0,0 @@ -package main - -import ( - "context" - "fmt" - - "github.com/nats-io/nats.go/jetstream" -) - -// **A part does not stop for a bucket it can work without** (novox/hq issue 1945, met twice on 2026-10-25: the builder of -// mesh-delivery #48 exited on a refused read of the gate's source, and its new reasks bucket would have done the same). A -// grant reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted -// yet: exiting for it stops the part until a person's hand. A bucket the watchman cannot do its job at all without still stops -// the start, naming the grant to look at; every other is said and left. (The same two functions are in each delivery part; -// they belong in mesh-sdk.) - -// bucketOpener is the part of the bus a start opens buckets with. -type bucketOpener interface { - KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error) -} - -// essentialBucket opens a bucket the watchman cannot work without; its error names the bucket and the grant. -func essentialBucket(ctx context.Context, js bucketOpener, name, does string) (jetstream.KeyValue, error) { - kv, err := js.KeyValue(ctx, name) - if err != nil { - return nil, essentialErr(name, does, err) - } - return kv, nil -} - -// essentialErr is the error of a bucket the part cannot work without: it names the bucket and the grant. -func essentialErr(name, does string, err error) error { - return fmt.Errorf("the bucket %s cannot be read (%v): the watchman cannot %s without it, so it stops; where the bus "+ - "refused it or did not answer, this module's grant of %s (its manifest's reads or state, issued with its credential) "+ - "has not reached the bus yet", name, err, does, name) -} - -// optionalBucket opens a bucket the watchman works without: any error is said with what is not heard, and nil answered. -func optionalBucket(ctx context.Context, js bucketOpener, name, without string, logf func(string, ...any)) jetstream.KeyValue { - kv, err := js.KeyValue(ctx, name) - if err != nil { - logf("the bucket %s cannot be read (%v): %s", name, err, without) - return nil - } - return kv -} diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index d343e0e..b154f77 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-watchman go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-watchman/go.sum b/modules/mesh-delivery-watchman/go.sum index bfba8a6..3c78612 100644 --- a/modules/mesh-delivery-watchman/go.sum +++ b/modules/mesh-delivery-watchman/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=