diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 0c7341d..3ba0133 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -86,11 +86,11 @@ var keptStates = []announce.State{ // buckets is every bucket the assigner opened, by name. type buckets map[string]jetstream.KeyValue -// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named, since a pass -// without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it +// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named with its grant (issue 1945), +// since a pass without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it // does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants // without it and takes no put further until it starts again (ADR 0546). -func open(ctx context.Context, js jetstream.JetStream, logf func(string, ...any)) (buckets, error) { +func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) { out := buckets{} for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) { kv, err := js.KeyValue(ctx, name) @@ -99,7 +99,7 @@ func open(ctx context.Context, js jetstream.JetStream, logf func(string, ...any) continue } if err != nil { - return nil, fmt.Errorf("the bucket %s cannot be opened: %w", name, err) + return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err) } out[name] = kv } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go index 3c346ab..75d7411 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go @@ -73,7 +73,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) { switch { case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")): t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged) - case !c.starts && (err == nil || !strings.Contains(err.Error(), c.missing)): + case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err) } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 1bbe91f..32a1dcc 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "fmt" "strings" "time" @@ -35,6 +36,9 @@ type passer struct { neither map[string]neitherOne // acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket). acts *putActs + // noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is + // said in the journal when it appears or changes, and its clearing when the node has a want again; one line per change. + noWant map[string]string } func (p *passer) clock() time.Time { @@ -148,6 +152,7 @@ func (p *passer) wants(ctx context.Context) { } out := assign.Wants(m, nodes, wanted[name], last) gatherNeither(neither, name, out) + p.sayNoWant(name, m.Nodes, wanted[name], out) if first { startClears(stale, m, out) if fp != "" && !m.Held && out.Held == nil && !out.Lifted { @@ -186,6 +191,42 @@ func (p *passer) wants(ctx context.Context) { } } +// sayNoWant says in the journal why a node of module gets no want after this pass, from the rule the outcome names for it +// (issue 1945: a new assignment whose trunk build was not done was held back with nothing said), once per reason, and says +// when the node has a want again. A node with a live want, written before and kept or written in this pass, has one. +func (p *passer) sayNoWant(module string, nodes []string, before map[string]delivery.Want, out assign.Outcome) { + if p.noWant == nil { + p.noWant = map[string]string{} + } + assigned := map[string]bool{} + for _, node := range nodes { + assigned[node] = true + key := node + "." + module + w, written := out.Wants[node] + if !written { + w, written = before[node] + } + reason := "" + if h, held := out.Rules[node]; held && (!written || w.Removed) { + reason = fmt.Sprintf("rule %d: %s", h.Rule, h.Words) + } + last, said := p.noWant[key] + switch { + case reason != "" && reason != last: + p.logf("%s on %s gets no want: %s", module, node, reason) + p.noWant[key] = reason + case reason == "" && said && written && !w.Removed: + p.logf("%s on %s has a want", module, node) + delete(p.noWant, key) + } + } + for key := range p.noWant { + if node, m, _ := strings.Cut(key, "."); m == module && !assigned[node] { + delete(p.noWant, key) // no longer assigned there: its removed want says so + } + } +} + // neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back. type neitherOne struct { node, module string diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 0458ebc..ff438f1 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -380,6 +380,31 @@ func TestThePassGathersTheModulesOnNeitherList(t *testing.T) { if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant { t.Errorf("gathered = %+v", got) } + // 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and + // when it has a want again: one line per change, not per pass. + p := newPasser(buckets{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC)) + var logged []string + p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) } + notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleBuildDone, Words: "the build of rofi at 4e7b118e is not done"}}} + wanted := assign.Outcome{Wants: map[string]delivery.Want{"shanks": {FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}}} + for i, c := range []struct { + before map[string]delivery.Want + out assign.Outcome + line string // the line said in this pass, "" for none + }{ + {nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is not done"}, + {nil, notDone, ""}, + {map[string]delivery.Want{"shanks": {Removed: true}}, notDone, ""}, + {nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"}, + {nil, wanted, "rofi on shanks has a want"}, + {map[string]delivery.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""}, + } { + logged = nil + p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out) + if got := strings.Join(logged, "\n"); got != c.line { + t.Errorf("pass %d said %q, want %q", i, got, c.line) + } + } } // A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index 893c786..df8760f 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index 2d6d7be..8492faa 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -6,6 +6,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv4 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index 8d0daff..e275c59 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -333,8 +333,18 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st // the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2, // held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither // list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour). - if h, there := n.Haves[node][m.Name]; there && h.Phase != delivery.PhaseRemoved && !held { + h, there := n.Haves[node][m.Name] + switch { + case there && h.Phase != delivery.PhaseRemoved && !held: out.Neither[node] = Holding{Rule: rule, Words: words} + case had && (!there || h.Phase == delivery.PhaseRemoved): + // A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's + // assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was + // would have its node agent remove the module and stop holding its composition's resources back. + next = delivery.Want{Removed: true, Words: NoWantWords(rule, words)} + if !delivery.SameWant(normal(before), normal(next)) { + out.Wants[node] = next + } } continue } @@ -370,6 +380,11 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st return out } +// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945). +func NoWantWords(rule int, words string) string { + return fmt.Sprintf("no want: rule %d: %s", rule, words) +} + // failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or // moving). A failed health while it // backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index 69deb19..25ae9d1 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -867,6 +867,33 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) { if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone { t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither) } + // 1945 (the assigner holds a first want back silently): with no build, a node whose want was removed and that does not run + // the module has the reason written into its removed want, once; a node with no key gets no want written, and a node that + // runs the module is on neither list, its removed want left as it is. + gone := delivery.Want{Removed: true, Words: "screen-lock is no longer assigned to shanks"} + for _, c := range []struct { + name string + before map[string]delivery.Want + want map[string]delivery.Want + }{ + {"no key: nothing written", nil, map[string]delivery.Want{}}, + {"a removed want: it says why", map[string]delivery.Want{"shanks": gone}, + map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}}, + {"a removed want that says why already: nothing written", map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}, + map[string]delivery.Want{}}, + {"a node that runs it: on neither list, its removed want left", map[string]delivery.Want{"g14": gone}, map[string]delivery.Want{}}, + } { + out := Wants(m, n, c.before, nil) + if len(out.Wants) != len(c.want) { + t.Errorf("%s: wants = %+v, want %+v", c.name, out.Wants, c.want) + continue + } + for node, w := range c.want { + if !delivery.SameWant(normal(out.Wants[node]), w) { + t.Errorf("%s: want of %s = %+v, want %+v", c.name, node, out.Wants[node], w) + } + } + } m.FP = "" if out := Wants(m, n, nil, nil); len(out.Neither) != 0 { t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go new file mode 100644 index 0000000..acc2bfe --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -0,0 +1,93 @@ +// Command mesh-delivery-builder-tools serves the builder's tools (novox/hq issue 1945): the runtime launches it for a call. +// builder_stuck reads the builder's builds and reasks states and the build modules' steps through the builder's state and +// reads; builder_ask_again writes one value of the reasks state, which the builder watches and answers. The builder is the +// other program: it serves no tool (ADR 0437 §4). +package main + +import ( + "encoding/json" + "fmt" + "os" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-builder/internal/build" +) + +// Tools are the builder's two tools: stuck reads, askAgain writes the reasks value. +func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, now func() time.Time) []stdio.Tool { + str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + return []stdio.Tool{ + {Name: build.StuckTool, + Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " + + "waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " + + "answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " + + "seat workers' delivery counts are not readable by the builder. With module, that module's keys only. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "only this module's keys"}}}, + Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }}, + {Name: build.AskAgainTool, + Description: "Ask a waiting build key's asks again now, under new ids the key then names, as the builder does by itself for an ask its build " + + "seat's stream gave up on. The builder answers in the reasks state, which " + build.StuckTool + " shows beside the key; a key no ask of " + + "which waits is answered so and nothing is asked. An ask whose step already runs on a node is asked again too: it is then " + + "built twice, the first run's work wasted and its end read by nobody, never a wrong key; builder_stuck shows whether a step runs.", + Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module's name"}, + "fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:, as builder_stuck shows it"}, + "why": map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + key, value, err := build.NewReAsk(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now()) + if err != nil { + return nil, err + } + if err := askAgain(key, value); err != nil { + return nil, fmt.Errorf("the ask could not be written to %s: %w", build.ReAsksBucket, err) + } + return map[string]any{"key": key, "asked": true, + "next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil + }}, + } +} + +func read(s stdio.KeptState) build.StateRead { + return build.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, time.Time, error) { + e, err := s.Get(key) + if err != nil || e == nil { + return nil, time.Time{}, err + } + return e.Value, written(s, key, e.Revision), nil + }} +} + +// written is when the key's revision was written, from its history, or zero. +func written(s stdio.KeptState, key string, revision uint64) time.Time { + h, err := s.History(key) + if err != nil { + return time.Time{} + } + for _, r := range h { + if r.Revision == revision { + return r.At + } + } + return time.Time{} +} + +func main() { + steps := map[string]build.StateRead{} + for _, bm := range build.StuckBuildModules { + steps[bm] = read(stdio.State(bm + ".steps")) + } + reasks := stdio.State(build.ReAsksState) + r := build.StuckReader{Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps} + put := func(key string, value []byte) error { + _, err := reasks.Put(key, json.RawMessage(value)) + return err + } + if err := stdio.Serve("", Tools(r, put, time.Now)); err != nil { + fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err) + os.Exit(1) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go index e5493a4..1c5afd8 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go @@ -8,57 +8,82 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-builder/internal/build" ) // heldOnTheBus says whether an ask a build key names is on its build seat's stream, or a build node wrote a step for it -// (issue 1910): read once, at the take-over of a key at start, never polled. +// (issue 1910), read once at the take-over of a key at start; and, at the builder's check bound, whether an ask can still +// be answered (issue 1945). Never polled. type heldOnTheBus struct{ js jetstream.JetStream } // Held is build.Builder's Held: a step key `.` in the build module's steps, or a message of the stream under // the ask's subject whose Nats-Msg-Id is the ask's id. func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error) { - module, err := delivery.BuildModuleOf(a.Language) + seen, err := h.seen(ctx, a, false) if err != nil { return false, err } + return seen.Step || seen.OnStream, nil +} + +// Gone is build.Builder's Gone: why the ask can no longer be answered (build.AskSeen.Why), or "". +func (h heldOnTheBus) Gone(ctx context.Context, a delivery.StepAsk) (string, error) { + seen, err := h.seen(ctx, a, true) + if err != nil { + return "", err + } + return seen.Why(), nil +} + +// seen reads what the bus holds of the ask; with later, a step of its build module started after the ask was put too, +// when the ask waits first in line. +func (h heldOnTheBus) seen(ctx context.Context, a delivery.StepAsk, later bool) (build.AskSeen, error) { + var seen build.AskSeen + module, err := delivery.BuildModuleOf(a.Language) + if err != nil { + return seen, err + } pattern, err := delivery.AskStepWatch(a.ID) if err != nil { - return false, err + return seen, err } steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module)) if err != nil { - return false, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err) + return seen, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err) } keys, err := steps.ListKeysFiltered(ctx, pattern) if err != nil { - return false, err + return seen, err } for range keys.Keys() { keys.Stop() - return true, nil // a node took it + seen.Step = true // a node took it + return seen, nil } subject, err := delivery.AskSubject(a.Language, a.Kind) if err != nil { - return false, err + return seen, err } // The seat's stream by its name (delivery.AskStream), as the broker grants the builder its information and its // messages by sequence and nothing wider (issue 1910). name, err := delivery.AskStream(a.Language) if err != nil { - return false, err + return seen, err } + seen.Stream = name stream, err := h.js.Stream(ctx, name) if errors.Is(err, jetstream.ErrStreamNotFound) { - return false, nil + return seen, nil } if err != nil { - return false, err + return seen, err } // A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read by sequence // instead, each a direct read that consumes nothing. info, err := stream.Info(ctx) if err != nil { - return false, err + return seen, err } for seq := info.State.FirstSeq; seq != 0 && seq <= info.State.LastSeq; seq++ { m, err := stream.GetMsg(ctx, seq) @@ -66,11 +91,37 @@ func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error continue // taken and acknowledged, or removed } if err != nil { - return false, err + return seen, err } - if m.Subject == subject && m.Header.Get(jetstream.MsgIDHeader) == a.ID { - return true, nil + if m.Subject != subject { + continue } + if m.Header.Get(jetstream.MsgIDHeader) == a.ID { + seen.OnStream, seen.PutAt = true, m.Time + break + } + seen.Ahead++ } - return false, nil + if !later || !seen.OnStream || seen.Ahead > 0 { + return seen, nil + } + // First in line and not taken: did a build node take any ask put after it? Its stream then no longer delivers it. + all, err := steps.ListKeys(ctx) + if err != nil { + return seen, err + } + defer all.Stop() + for k := range all.Keys() { + e, err := steps.Get(ctx, k) + if err != nil { + continue + } + s, err := delivery.DecodeStep(e.Value()) + if err != nil || !s.Started.After(seen.PutAt) { + continue // not a step (a node's pause key), or one started before the ask was put + } + seen.Later = &s + break + } + return seen, nil } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 77cbc22..1726dc5 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -13,7 +13,6 @@ import ( "encoding/json" "errors" "fmt" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "io" "log" "maps" @@ -25,6 +24,8 @@ import ( "syscall" "time" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -124,29 +125,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - bucket := func(name string) (jetstream.KeyValue, error) { - kv, err := js.KeyValue(ctx, name) - if err != nil { - return nil, fmt.Errorf("the bucket %s: %w", name, err) - } - return kv, nil - } - builds, err := bucket(delivery.BuilderBuildsBucket) + builds, err := process.EssentialBucket(ctx, js, "builder", delivery.BuilderBuildsBucket, "claim or write a build key") if err != nil { return err } - modules, err := bucket(delivery.ModulesBucket) + modules, err := process.EssentialBucket(ctx, js, "builder", delivery.ModulesBucket, "know any fingerprint to build") if err != nil { return err } - manifests, err := bucket(delivery.ManifestsBucket) - if err != nil { - return err - } - candidates, err := bucket(delivery.CandidatesBucket) + manifests, err := process.EssentialBucket(ctx, js, "builder", delivery.ManifestsBucket, "read what a module builds") if err != nil { return err } + // The candidates' fingerprints are built too (issue 1684): a builder that cannot read them builds the trunk's. + candidates := process.OptionalBucket(ctx, js, delivery.CandidatesBucket, "no candidate's fingerprint is built until the builder starts again; the trunk's are", logger.Printf) bound := defaultStepBound if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" { if bound, err = time.ParseDuration(v); err != nil { @@ -169,44 +161,36 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } defer calls.Close() b := &build.Builder{ - Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module}, - Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound}, - Held: heldOnTheBus{js}, + Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module}, + Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound}, + Held: heldOnTheBus{js}, + // An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound. + Gone: heldOnTheBus{js}, + CheckAfter: bound, Toolchains: tools, Manifests: manifestReader{manifests}, Logf: logger.Printf, Stored: storedVia(calls.Call), } // A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base - // revision (the gate's source state). Where the bucket is not there yet no candidate's fingerprint is built, and said. - // The source's revisions are watched with their history (a direct get of one revision is not what a read grants): a - // revision heard after the candidate that names it builds that candidate then. - trunks := &sourceHeads{} - var sw jetstream.KeyWatcher - switch kv, err := js.KeyValue(ctx, delivery.SourceBucket); { - case errors.Is(err, jetstream.ErrBucketNotFound): - logger.Printf("the bucket %s does not exist yet: no candidate's fingerprint is built until the builder starts again", delivery.SourceBucket) - case err != nil: - return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err) - default: - w, err := kv.Watch(ctx, delivery.SourceMainKey, jetstream.IncludeHistory()) - if err != nil { - return fmt.Errorf("the bucket %s cannot be watched: %w", delivery.SourceBucket, err) + // revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision + // is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it + // cannot be read the trunk's keys are built and the candidates wait (issue 1945). + errs := make(chan error, 4) + trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: b.TrunkMoved, errs: errs, logf: logger.Printf} + trunks.start() + b.Trunk = trunks.head + b.Said = func(e build.AskedAgain) { + if err := emitAskedAgain(context.WithoutCancel(ctx), js, cred, e); err != nil { + logger.Printf("%v", err) } - if sw, err = settled(ctx, w, func(e jetstream.KeyValueEntry) { trunks.put(e) }); err != nil { - return err - } - b.Trunk = trunks.head } // **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the // controller delivers from this builder's keys only while an instance says it writes references. A bucket not there // yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent // is asked since R4g, ADR 0524 §7). - switch kv, err := js.KeyValue(ctx, instancesBucket); { - case errors.Is(err, jetstream.ErrBucketNotFound): - logger.Printf("the bucket %s does not exist yet: the controller is not told that this builder writes references until it starts again", instancesBucket) - case err != nil: - return fmt.Errorf("the bucket %s: %w", instancesBucket, err) + switch kv := process.OptionalBucket(ctx, js, instancesBucket, "the controller is not told that this builder writes references until it starts again", logger.Printf); { + case kv == nil: default: says := newSayer(ctx, kv, cred.Node, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, build.ImageBuildModule}, logger.Printf) b.Answered = says.Answered @@ -286,23 +270,26 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro b.ForgetCandidate(e.Key()) } } - cw, err := settle(ctx, candidates, eachCandidate) - if err != nil { - return err + if candidates != nil { + cw, err := settle(ctx, candidates, eachCandidate) + if err != nil { + process.NotWatched(logger.Printf, delivery.CandidatesBucket, err, "no candidate's fingerprint is built until the builder starts again") + } else { + go func() { errs <- follow(ctx, cw, eachCandidate) }() + } + } + // A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as + // every other input. The builder asks again by itself without it. + if kv := process.OptionalBucket(ctx, js, build.ReAsksBucket, "builder_ask_again is not heard until the builder starts again; an ask that can no longer be answered is still asked again by itself", logger.Printf); kv != nil { + eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) } + if rw, err := settle(ctx, kv, eachReAsk); err != nil { + process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again") + } else { + go func() { errs <- follow(ctx, rw, eachReAsk) }() + } } logger.Printf("started on %s", cred.Node) - errs := make(chan error, 3) go func() { errs <- follow(ctx, w, each) }() - go func() { errs <- follow(ctx, cw, eachCandidate) }() - if sw != nil { - go func() { - errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { - if trunks.put(e) { - b.TrunkMoved(ctx) - } - }) - }() - } select { case <-ctx.Done(): b.Wait() diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go new file mode 100644 index 0000000..02afe10 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go @@ -0,0 +1,78 @@ +package main + +import ( + "context" + "fmt" + "sync" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// **A part does not stop for a bucket it can work without** (mesh-sdk process.EssentialBucket, OptionalBucket) (novox/hq issue 1945, met twice on 2026-10-25: the builder of +// mesh-delivery #48 exited on a refused read of the gate's source, and its reasks bucket would have done the same). A grant +// reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted yet: +// exiting for it stops every build until a person's hand. A bucket the builder cannot build at all without (its builds, +// the indexer's modules and manifests) still stops the start, naming the grant to look at; every other is said and left, +// and opened again at the next event that needs it where there is one. + +// trunkSource is the gate's source history the candidates' previews are merged onto (sourceHeads), opened at start and, +// while it cannot be, again at the next candidate that asks a trunk head: a builder without it builds the trunk's keys, +// and a candidate's preview waits, saying why. +type trunkSource struct { + ctx context.Context + js bucketOpener + heads *sourceHeads + moved func(context.Context) + errs chan<- error + logf func(string, ...any) + + mu sync.Mutex + opened bool +} + +// ensure opens and settles the source's watch once, and follows it; an error leaves it closed for the next ask. +func (t *trunkSource) ensure() error { + t.mu.Lock() + defer t.mu.Unlock() + if t.opened { + return nil + } + kv, err := t.js.KeyValue(t.ctx, delivery.SourceBucket) + if err != nil { + return fmt.Errorf("the bucket %s cannot be read (%v)", delivery.SourceBucket, err) + } + w, err := kv.Watch(t.ctx, delivery.SourceMainKey, jetstream.IncludeHistory()) + if err != nil { + return fmt.Errorf("the bucket %s cannot be watched (%v)", delivery.SourceBucket, err) + } + sw, err := settled(t.ctx, w, func(e jetstream.KeyValueEntry) { t.heads.put(e) }) + if err != nil { + return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err) + } + t.opened = true + go func() { + t.errs <- follow(t.ctx, sw, func(e jetstream.KeyValueEntry) { + if t.heads.put(e) && t.moved != nil { + t.moved(t.ctx) + } + }) + }() + return nil +} + +// start opens the source at start; one that cannot be is said, and the builder goes on. +func (t *trunkSource) start() { + if err := t.ensure(); err != nil { + t.logf("%v: the trunk's keys are built, and a candidate's preview waits until the source can be read, tried again at its next candidate", err) + } +} + +// head is Builder.Trunk: the head of repository at source revision rev, opening the source first if it is not yet. +func (t *trunkSource) head(ctx context.Context, rev uint64, repository string) (string, error) { + if err := t.ensure(); err != nil { + return "", err + } + return t.heads.head(ctx, rev, repository) +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go new file mode 100644 index 0000000..48bcd29 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go @@ -0,0 +1,75 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "mesh-delivery-builder/internal/build" +) + +// askAgainer is the part of the builder a reasks value asks through. +type askAgainer interface { + AskAgain(key, why string) []string +} + +// reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the +// key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value +// answered already (this builder's own write) or deleted is left. +func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) { + if e.Operation() != jetstream.KeyValuePut { + return + } + r, err := build.DecodeReAsk(e.Value()) + if err != nil { + logf("%s.%s holds what is not an ask to ask again, left: %v", build.ReAsksState, e.Key(), err) + return + } + if r.Answer != "" { + return + } + asked := b.AskAgain(e.Key(), r.Why) + if len(asked) == 0 { + r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands" + } else { + r.Answer = "asked again: the " + strings.Join(asked, ", ") + " ask, under a new id the key names" + } + r.Answered = time.Now().UTC() + logf("%s: asked again by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer) + data, err := json.Marshal(r) + if err != nil { + return + } + if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err) + } +} + +// emitAskedAgain says the builder's event build.asked-again (issue 1945): mesh.mod..event.build.asked-again, +// with the SDK's envelope headers and the new ask's id as the bus's dedup id. +func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred credential, e build.AskedAgain) error { + data, err := json.Marshal(e) + if err != nil { + return err + } + msg := nats.NewMsg("mesh.mod." + cred.Module + ".event." + build.AskedAgainEvent) + id := "asked-again-" + e.New + msg.Header.Set("x-event-id", id) + msg.Header.Set("x-source", cred.Module) + msg.Header.Set("x-node", cred.Node) + msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Data = data + pub, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil { + return fmt.Errorf("saying %s for %s: %w", build.AskedAgainEvent, e.Key, err) + } + return nil +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing.go index 3d6decf..9a06cec 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing.go @@ -193,7 +193,9 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) } w, err := settle(ctx, kv, each) if err != nil { - return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err) + // Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read. + logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err) + return errs, nil } if gate.Settled() { changed(ctx) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go index 28fccc4..4a0e7ca 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go @@ -3,11 +3,19 @@ package main // The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940). import ( + "bytes" + "context" + "errors" "fmt" + "log" "strings" "testing" "time" + "git.novox.be/novox/mesh-sdk/go/process" + + "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) @@ -89,6 +97,61 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) { if n := len(set.Entries("typescript")); n != 2 { t.Fatalf("novox and shanks are left; got %d entries", n) } + + // 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every + // toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without + // is an error naming the bucket and its grant. + var logs bytes.Buffer + logger := log.New(&logs, "", 0) + refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")} + if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil { + t.Errorf("a refused watch of the assignments stopped the start: %v", err) + } + if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil { + t.Errorf("a refused toolchains bucket stopped the start: %v", err) + } + if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") { + t.Errorf("the refusals were not said: %q", logs.String()) + } + if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil || + !strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") { + t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err) + } + // The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk + // head is told why and opens it again then. + src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf} + src.start() + if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) { + t.Errorf("a trunk head with the source refused: %v, want why", err) + } + if src.opened { + t.Error("a refused source counts as opened, so it is never opened again") + } +} + +// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err. +type refusingBus struct { + refuseOpen bool + err error +} + +func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { + if r.refuseOpen { + return nil, r.err + } + return refusedWatch{err: r.err}, nil +} + +type refusedWatch struct { + jetstream.KeyValue + err error +} + +func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + return nil, r.err +} +func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + return nil, r.err } // A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go index 69faff2..58ae514 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go @@ -4,11 +4,13 @@ import ( "context" "errors" "fmt" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "log" "strings" "sync" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -62,11 +64,14 @@ func (w *toolchainWatch) ensure(module string) error { w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module) return nil case err != nil: - return fmt.Errorf("the bucket %s: %w", name, err) + // Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report. + w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module) + return nil } watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) }) if err != nil { - return err + process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module)) + return nil } w.watched[module] = true go func() { diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index 615cce0..fb89b4d 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index d648ca0..2dfa4fd 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-builder/internal/build/askagain.go b/modules/mesh-delivery-builder/internal/build/askagain.go new file mode 100644 index 0000000..980b5f6 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/askagain.go @@ -0,0 +1,235 @@ +package build + +import ( + "context" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up +// once it was delivered as often as its worker may (delivery.AskMaxDeliver), and a message given up on, dropped or +// terminated with no step written is answered by no build node: the lane waiting for its step would wait for ever, and on +// restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound), +// a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new +// id, written into the key, said in its log and as the event build.asked-again, at most Askings times; then the key is +// failed with why. A person asks the same by hand with the tool builder_ask_again (AskAgain). + +// DefaultAskings is how many times a build module's part of a build is asked, the first included, when Builder.Askings is 0. +const DefaultAskings = 3 + +// AskedAgainEvent is the event the builder says when it asked an ask again: mesh.mod.mesh-delivery-builder.event.. +const AskedAgainEvent = "build.asked-again" + +// AskedAgain is the body of AskedAgainEvent. +type AskedAgain struct { + Key string `json:"key"` + Module string `json:"module"` + FP string `json:"fp"` + BuildModule string `json:"build_module"` + Language string `json:"language"` + Old string `json:"old"` + New string `json:"new"` + Asking int `json:"asking"` + Askings int `json:"askings"` + Why string `json:"why"` + At time.Time `json:"at"` +} + +// AskSeen is what the bus says of an ask whose step has not ended, as the builder may read it: its build seat's stream +// (information and messages by sequence) and its build module's steps. The builder may not read the seat workers' +// delivery counts (granted to the delivery view alone), so a message given up on is told by what the workers did after it. +type AskSeen struct { + Stream string + // Step is whether a build node wrote a step for the ask (running: the step bound of buildrun.Asker judges it). + Step bool + // OnStream is whether the stream still holds the ask's message; PutAt is when it was put, Ahead how many messages of + // its subject wait before it. + OnStream bool + PutAt time.Time + Ahead int + // Later is a step of the same build module that started after the ask was put, when one did. + Later *delivery.Step +} + +// Why is why the ask can no longer be answered, or "" when it can: it has a step, waits behind others of its kind, or no +// build node took any ask since it was put (the nodes are busy or paused: a backlog, never a fault, ADR 0525 §9). +func (s AskSeen) Why() string { + switch { + case s.Step: + return "" + case !s.OnStream: + return fmt.Sprintf("no build node wrote a step for it and %s no longer holds it: it was given up on or dropped", s.Stream) + case s.Ahead > 0 || s.Later == nil: + return "" + default: + return fmt.Sprintf("it waits first in line on %s since %s while %s took a later ask at %s: its stream no longer delivers it (given up after its most deliveries)", + s.Stream, s.PutAt.UTC().Format(time.RFC3339), s.Later.Node, s.Later.Started.UTC().Format(time.RFC3339)) + } +} + +func (b *Builder) askings() int { + if b.Askings > 0 { + return b.Askings + } + return DefaultAskings +} + +var askingWords = regexp.MustCompile(`asked (\S+) again \(asking (\d+)\)`) + +// askingOf is the asking a key's step words name for a build module's ask, 1 when none. +func askingOf(step, buildModule string) int { + for _, m := range askingWords.FindAllStringSubmatch(step, -1) { + if m[1] == buildModule { + if n, err := strconv.Atoi(m[2]); err == nil && n > 0 { + return n + } + } + } + return 1 +} + +// AskAgain asks the waiting asks of a build key again now, under new ids (the tool builder_ask_again): it answers the +// languages whose lane waits, none when no lane of this builder waits on the key. +func (b *Builder) AskAgain(key, why string) []string { + b.init() + b.mu.Lock() + defer b.mu.Unlock() + var asked []string + for language, c := range b.waiting[key] { + select { + case c <- why: + default: // one is already asked + } + asked = append(asked, language) + } + return asked +} + +// Waiting are the build keys whose lanes wait for a step now, each with its languages. +func (b *Builder) Waiting() map[string][]string { + b.init() + b.mu.Lock() + defer b.mu.Unlock() + out := map[string][]string{} + for key, ls := range b.waiting { + for l := range ls { + out[key] = append(out[key], l) + } + } + return out +} + +func (b *Builder) listen(key, language string) chan string { + b.init() + c := make(chan string, 1) + b.mu.Lock() + if b.waiting[key] == nil { + b.waiting[key] = map[string]chan string{} + } + b.waiting[key][language] = c + b.mu.Unlock() + return c +} + +func (b *Builder) unlisten(key, language string, c chan string) { + b.mu.Lock() + if b.waiting[key][language] == c { + delete(b.waiting[key], language) + if len(b.waiting[key]) == 0 { + delete(b.waiting, key) + } + } + b.mu.Unlock() +} + +var ( + errCheck = errors.New("the check bound passed") + errAgain = errors.New("asked again") +) + +type byHand struct{ why string } + +func (h byHand) Error() string { return "asked again by hand: " + h.why } + +// answer waits for the step of the lane's ask, and asks it again while it can no longer be answered (Gone) or a person +// asked it again (AskAgain); it answers the step and the ask that has it. +func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, asked *askNotes, module, fp string) (delivery.Step, delivery.StepAsk, error) { + key, _ := delivery.BuilderBuildKey(module, fp) + again := b.listen(key, ask.Language) + defer b.unlisten(key, ask.Language, again) + asking := askingOf(asked.step, l.module) + for { + step, why, err := b.waitOnce(ctx, ask, again) + switch { + case errors.Is(err, errCheck): + continue + case errors.Is(err, errAgain): + if asking >= b.askings() { + return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why) + } + asking++ + old := ask.ID + ask.ID, ask.At = buildstep.NewID(), time.Now() + if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("asked %s again (asking %d): %s", l.module, asking, why)); err != nil { + return step, ask, fmt.Errorf("the ask of %s asked again could not be written to the key: %w", l.module, err) + } + if err := b.Asks.Put(ctx, ask); err != nil { + return step, ask, err + } + b.logf("%s: the ask %s of %s is asked again as %s (asking %d of %d): %s", key, old, l.module, ask.ID, asking, b.askings(), why) + if b.Said != nil { + b.Said(AskedAgain{Key: key, Module: module, FP: fp, BuildModule: l.module, Language: ask.Language, Old: old, New: ask.ID, + Asking: asking, Askings: b.askings(), Why: why, At: ask.At}) + } + default: + return step, ask, err + } + } +} + +// waitOnce waits for the ask's step until it ends, a person asks it again, or the check bound passes: then it asks Gone +// and answers errAgain with why when the ask can no longer be answered, errCheck when it waits on. +func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-chan string) (delivery.Step, string, error) { + hand, cancel := context.WithCancelCause(ctx) + defer cancel(nil) + go func() { + select { + case why := <-again: + cancel(byHand{why}) + case <-hand.Done(): + } + }() + wait := hand + if b.Gone != nil && b.CheckAfter > 0 { + var stop context.CancelFunc + wait, stop = context.WithTimeoutCause(hand, b.CheckAfter, errCheck) + defer stop() + } + step, err := b.Asks.Wait(wait, ask) + if err == nil || ctx.Err() != nil { + return step, "", err + } + var h byHand + if errors.As(context.Cause(hand), &h) { + return step, strings.TrimSpace("asked again by hand: " + h.why), errAgain + } + if errors.Is(context.Cause(wait), errCheck) { + why, gerr := b.Gone.Gone(ctx, ask) + if gerr != nil { + b.logf("whether the ask %s can still be answered cannot be read, so it is waited for: %v", ask.ID, gerr) + return step, "", errCheck + } + if why == "" { + return step, "", errCheck + } + return step, why, errAgain + } + return step, "", err +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 492cae5..6283da8 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -56,6 +56,18 @@ type Builder struct { // Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a // candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why. Trunk func(ctx context.Context, rev uint64, repository string) (string, error) + // Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the + // ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked + // again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it. + Gone interface { + Gone(ctx context.Context, a delivery.StepAsk) (string, error) + } + CheckAfter time.Duration + // Askings is how many times one build module's part of a build is asked, the first included, before its key is failed + // with why; DefaultAskings when 0. + Askings int + // Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody. + Said func(AskedAgain) once sync.Once sem chan struct{} @@ -75,6 +87,8 @@ type Builder struct { // held are the done keys' references the store answered held, by ` `, asked once per process: a // blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937). held map[string]bool + // waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them. + waiting map[string]map[string]chan string } type planned struct { @@ -91,7 +105,7 @@ func (b *Builder) init() { b.sem = make(chan struct{}, limit) b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{} b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{} - b.published, b.held = map[string]bool{}, map[string]bool{} + b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{} }) } @@ -673,12 +687,14 @@ type askNotes struct { b *Builder cl *Claim resumed map[string]string + // step is the step words of a key taken over at start: an ask asked again says its asking there (askingOf). + step string } func (b *Builder) notes(cl *Claim) *askNotes { n := &askNotes{b: b, cl: cl} if cl.Build.State == delivery.BuildBuilding && len(cl.Build.Asks) > 0 { - n.resumed = maps.Clone(cl.Build.Asks) + n.resumed, n.step = maps.Clone(cl.Build.Asks), cl.Build.Step } return n } @@ -776,7 +792,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del return } } - step, err := b.Asks.Wait(ctx, ask) + step, ask, err := b.answer(ctx, l, ask, asked, module, m.FP2) if err != nil { l.err = fmt.Errorf("%s: %w", l.module, err) return diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index f4ad7b2..17d5cf4 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -616,27 +616,198 @@ func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // re } } -// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked -// again; the key is done when a node takes it. -func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { - r := newRig(t, goBundle) - r.asks.Answer = nil - r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond} - r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) - for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) == 0; time.Sleep(time.Millisecond) { +// goneAsks answers Gone from a function of the ask, and counts how often it was asked. +type goneAsks struct { + mu sync.Mutex + n int + why func(a delivery.StepAsk) string +} + +func (g *goneAsks) Gone(_ context.Context, a delivery.StepAsk) (string, error) { + g.mu.Lock() + g.n++ + g.mu.Unlock() + return g.why(a), nil +} + +func (g *goneAsks) asked() int { + g.mu.Lock() + defer g.mu.Unlock() + return g.n +} + +// waitForAsks waits until n asks were put. +func waitForAsks(t *testing.T, r *rig, n int) []delivery.StepAsk { + t.Helper() + for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) < n; time.Sleep(time.Millisecond) { if time.Now().After(deadline) { - t.Fatal("no ask was put") + t.Fatalf("%d asks put, want %d", len(r.asks.Asks()), n) } } - time.Sleep(100 * time.Millisecond) - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding { - t.Fatalf("the key = %+v while its ask waits, want building", v) - } - r.asks.End(r.answer(r.asks.Asks()[0])) - r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 { - t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks())) + return r.asks.Asks() +} + +// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked +// again; the key is done when a node takes it. Issue 1945: an ask that can no longer be answered (its stream gave it up) +// is asked again under a new id, said, at most Askings times; a person asks it again by hand the same way. +func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { + t.Run("waits in its stream: never asked again", func(t *testing.T) { + r := newRig(t, goBundle) + r.asks.Answer = nil + r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond} + gone := &goneAsks{why: func(delivery.StepAsk) string { return "" }} + r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + waitForAsks(t, r, 1) + time.Sleep(100 * time.Millisecond) + if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding { + t.Fatalf("the key = %+v while its ask waits, want building", v) + } + if gone.asked() == 0 { + t.Fatal("whether the ask can still be answered was never read at the check bound") // 1945 + } + r.asks.End(r.answer(r.asks.Asks()[0])) + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 { + t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks())) + } + }) + for _, c := range []struct { + name string + askings int + // step is the key's step words at start when it names an ask (a key taken over), "" for a fresh build. + step string + byHand bool + always bool // Gone says every ask cannot be answered, not only the first + puts int + done bool + asking int // the asking the event says, 0 when none is said + }{ + {name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2}, + {name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2}, + {name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2}, + {name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0}, + } { + t.Run(c.name, func(t *testing.T) { // 1945 + r := newRig(t, goBundle) + r.asks.Answer = nil + r.b.Askings = c.askings + var said []AskedAgain + var mu sync.Mutex + r.b.Said = func(e AskedAgain) { mu.Lock(); said = append(said, e); mu.Unlock() } + first := "" + if c.step != "" { + first = "mvl0frv18d1960" + data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD, + Step: c.step, Asks: map[string]string{delivery.AskGo: first}}) + r.st.vals[r.key] = data + } + if !c.byHand { + r.b.Gone = &goneAsks{why: func(a delivery.StepAsk) string { + if c.always || a.ID == first || first == "" { + return "it waits first in line while a node took a later ask" + } + return "" + }} + r.b.CheckAfter = 5 * time.Millisecond + } + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + if c.byHand { + waitForAsks(t, r, 1) + for deadline := time.Now().Add(2 * time.Second); len(r.b.AskAgain(r.key, "stuck since 19:06")) == 0; time.Sleep(time.Millisecond) { + if time.Now().After(deadline) { + t.Fatal("no lane of the key waits") + } + } + } + if c.done { + asks := waitForAsks(t, r, c.puts) + if first == "" && !c.byHand { + first = asks[0].ID + } + last := asks[len(asks)-1] + if v := valueOf(t, r.st, r.key); v.Asks[delivery.AskGo] != last.ID || !strings.Contains(v.Step, "again (asking 2)") { + t.Fatalf("the key = %+v, want building naming the ask asked again %s", v, last.ID) + } + r.asks.End(r.answer(last)) + } + r.b.Wait() + v := valueOf(t, r.st, r.key) + if n := len(r.asks.Asks()); n != c.puts { + t.Errorf("%d asks put, want %d", n, c.puts) + } + if c.done && v.State != delivery.BuildDone { + t.Errorf("the key = %+v, want done from the ask asked again", v) + } + if !c.done && (v.State != delivery.BuildFailed || !strings.Contains(v.Error, "could not be answered and was asked")) { + t.Errorf("the key = %+v, want failed saying the ask could not be answered", v) + } + mu.Lock() + defer mu.Unlock() + if c.asking > 0 && (len(said) != 1 || said[0].Asking != c.asking || said[0].New == said[0].Old || said[0].Key != r.key || said[0].Why == "") { + t.Errorf("said %+v, want once at asking %d", said, c.asking) + } + }) } + t.Run("what the bus says of an ask", func(t *testing.T) { // 1945 + put := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC) + later := &delivery.Step{Node: "g14", Started: put.Add(time.Minute)} + for _, c := range []struct { + seen AskSeen + want string // a part of the why, "" when it can be answered + }{ + {AskSeen{Stream: "S", Step: true}, ""}, + {AskSeen{Stream: "S"}, "no longer holds it"}, + {AskSeen{Stream: "S", OnStream: true, PutAt: put, Later: later}, "no longer delivers it"}, + {AskSeen{Stream: "S", OnStream: true, PutAt: put, Ahead: 2, Later: later}, ""}, + {AskSeen{Stream: "S", OnStream: true, PutAt: put}, ""}, + } { + got := c.seen.Why() + if (c.want == "") != (got == "") || !strings.Contains(got, c.want) { + t.Errorf("%+v: why %q, want %q", c.seen, got, c.want) + } + } + }) + t.Run("the tool builder_stuck", func(t *testing.T) { // 1945 + at := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC) + state := func(vals map[string]string) StateRead { + return StateRead{Keys: func() ([]string, error) { + var ks []string + for k := range vals { + ks = append(ks, k) + } + return ks, nil + }, Get: func(k string) (json.RawMessage, time.Time, error) { + if v, ok := vals[k]; ok { + return json.RawMessage(v), at, nil + } + return nil, time.Time{}, nil + }} + } + key, _ := delivery.BuilderBuildKey("m", fp1) + done, _ := delivery.BuilderBuildKey("d", fp1) + building, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD, + Step: "asked mesh-build-go again (asking 2): x", Asks: map[string]string{delivery.AskGo: "mvo53jcm70b5f7", delivery.AskImage: "mvo53jcm70b5f8"}}) + finished, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: manifestD, Step: "built"}) + running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at}) + _, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at) + r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}), + Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})}, + Now: func() time.Time { return at.Add(time.Hour) }} + out, err := r.Stuck("") + if err != nil { + t.Fatal(err) + } + stuck := out["stuck"].([]Stuck) + if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" { + t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck) + } + a := stuck[0].Asks + if len(a) != 2 || a[0].Language != delivery.AskGo || a[0].Asking != 2 || a[0].Step != "none" || a[0].Asked.IsZero() || + a[1].Asking != 1 || !strings.HasPrefix(a[1].Step, "running on g14") { + t.Fatalf("asks = %+v, want go at asking 2 with no step and image at asking 1 running on g14", a) + } + }) } // heldAsks says an ask is held when its id is among held. diff --git a/modules/mesh-delivery-builder/internal/build/stuck.go b/modules/mesh-delivery-builder/internal/build/stuck.go new file mode 100644 index 0000000..5bcaf0c --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/stuck.go @@ -0,0 +1,200 @@ +package build + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" +) + +// The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads +// every key that waits on an ask, builder_ask_again asks a key's build again now. The tools bundle serves them through +// the tool runner; the builder's process serves no tool (ADR 0437 §4), so builder_ask_again writes the ask into the +// builder's state reasks, which the builder watches and answers there. +const ( + StuckTool = "builder_stuck" + AskAgainTool = "builder_ask_again" + // ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key. + ReAsksState = "reasks" + ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState +) + +// ReAsk is a value of the reasks state: why a person asked the key's build again, who and when, and, once the builder +// read it, its answer. +type ReAsk struct { + V int `json:"v"` + Why string `json:"why"` + By string `json:"by,omitempty"` + At time.Time `json:"at"` + Answer string `json:"answer,omitempty"` + Answered time.Time `json:"answered,omitzero"` +} + +// DecodeReAsk reads a reasks value. +func DecodeReAsk(data []byte) (ReAsk, error) { + var r ReAsk + if err := json.Unmarshal(data, &r); err != nil { + return r, err + } + if r.V != 1 { + return r, fmt.Errorf("a reasks value of version %d, not 1", r.V) + } + return r, nil +} + +// NewReAsk is the value builder_ask_again writes for a module's fingerprint: its key and the value. +func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) { + key, err := delivery.BuilderBuildKey(module, fp) + if err != nil { + return "", nil, err + } + if strings.TrimSpace(why) == "" { + return "", nil, errors.New("say why the build is asked again (why)") + } + data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC()}) + return key, data, err +} + +// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known). +type StateRead struct { + Keys func() ([]string, error) + Get func(key string) (json.RawMessage, time.Time, error) +} + +// StuckAsk is one ask a key waits on. +type StuckAsk struct { + Language string `json:"language"` + BuildModule string `json:"build_module"` + ID string `json:"id"` + Asking int `json:"asking"` + // Asked is when the ask was put, read from its id; Step is what its build module's steps hold of it: none, or its + // state on its node since its start. + Asked time.Time `json:"asked,omitzero"` + Step string `json:"step"` +} + +// Stuck is one build key that waits on an ask. +type Stuck struct { + Key string `json:"key"` + Module string `json:"module"` + FP string `json:"fp"` + State string `json:"state"` + Step string `json:"step"` + By string `json:"by"` + Since time.Time `json:"since,omitzero"` + Age string `json:"age,omitempty"` + Asks []StuckAsk `json:"asks"` + ReAsk *ReAsk `json:"reask,omitempty"` + Unasked bool `json:"unasked,omitempty"` + NoneStep int `json:"asks_with_no_step"` +} + +// StuckReader reads what builder_stuck answers. +type StuckReader struct { + Builds StateRead + ReAsks StateRead + // Steps are the build modules' steps states, by build module. + Steps map[string]StateRead + Now func() time.Time +} + +// Stuck is every key building or queued with its asks, oldest first; with module, that module's only. +func (r StuckReader) Stuck(module string) (map[string]any, error) { + keys, err := r.Builds.Keys() + if err != nil { + return nil, err + } + now := time.Now + if r.Now != nil { + now = r.Now + } + stepKeys := map[string][]string{} + for bm, s := range r.Steps { + ks, err := s.Keys() + if err != nil { + return nil, fmt.Errorf("the steps of %s: %w", bm, err) + } + stepKeys[bm] = ks + } + out := []Stuck{} + for _, key := range keys { + mod, fp, err := delivery.ParseBuilderBuildKey(key) + if err != nil || (module != "" && mod != module) { + continue + } + raw, at, err := r.Builds.Get(key) + if err != nil || raw == nil { + continue + } + v, err := delivery.DecodeBuilderBuild(raw) + if err != nil || (v.State != delivery.BuildBuilding && v.State != delivery.BuildQueued) { + continue + } + s := Stuck{Key: key, Module: mod, FP: fp, State: v.State, Step: v.Step, By: v.By, Since: at, Asks: []StuckAsk{}} + if !at.IsZero() { + s.Age = now().Sub(at).Round(time.Second).String() + } + for language, id := range v.Asks { + bm, _ := delivery.BuildModuleOf(language) + a := StuckAsk{Language: language, BuildModule: bm, ID: id, Asking: askingOf(v.Step, bm), Step: "none"} + if t, ok := idTime(id); ok { + a.Asked = t + } + if st, ok := r.Steps[bm]; ok { + for _, k := range stepKeys[bm] { + if !strings.HasSuffix(k, "."+id) { + continue + } + if raw, _, err := st.Get(k); err == nil && raw != nil { + if step, err := delivery.DecodeStep(raw); err == nil { + a.Step = fmt.Sprintf("%s on %s since %s", step.State, step.Node, step.Started.UTC().Format(time.RFC3339)) + } + } + } + } + if a.Step == "none" { + s.NoneStep++ + } + s.Asks = append(s.Asks, a) + } + sort.Slice(s.Asks, func(i, j int) bool { return s.Asks[i].Language < s.Asks[j].Language }) + s.Unasked = v.State == delivery.BuildBuilding && len(v.Asks) == 0 + if r.ReAsks.Get != nil { + if raw, _, err := r.ReAsks.Get(key); err == nil && raw != nil { + if ra, err := DecodeReAsk(raw); err == nil { + s.ReAsk = &ra + } + } + } + out = append(out, s) + } + sort.Slice(out, func(i, j int) bool { + if !out[i].Since.Equal(out[j].Since) { + return out[i].Since.Before(out[j].Since) + } + return out[i].Key < out[j].Key + }) + return map[string]any{"stuck": out, "count": len(out), + "delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}, nil +} + +// StuckBuildModules are the build modules whose steps builder_stuck reads. +var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule} + +// idTime is when a step id was made (buildstep.NewID: the time in milliseconds in base 36, then six hex digits). +func idTime(id string) (time.Time, bool) { + if len(id) <= 6 { + return time.Time{}, false + } + ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64) + if err != nil || ms <= 0 { + return time.Time{}, false + } + return time.UnixMilli(ms).UTC(), true +} diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 27e44e9..7034836 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -23,18 +23,21 @@ type state struct { } type artifact struct { - Name string `json:"name"` - Kind string `json:"kind"` - Language string `json:"language"` - System string `json:"system"` - From string `json:"from"` - Binary string `json:"binary"` + Name string `json:"name"` + Kind string `json:"kind"` + Language string `json:"language"` + System string `json:"system"` + From string `json:"from"` + Binary string `json:"binary"` + Loads []string `json:"loads"` } type manifest struct { Module string `json:"module"` Version string `json:"version"` Slug string `json:"slug"` + Tools []string `json:"tools"` + Emits []string `json:"emits"` Consumes []string `json:"consumes"` State []state `json:"state"` Reads []string `json:"reads"` @@ -84,7 +87,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if m.Module != "mesh-delivery-builder" { t.Errorf("module = %q", m.Module) } - if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory + if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory t.Errorf("state = %+v, want %+v", m.State, want) } // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and @@ -112,7 +115,14 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if m.SecretsOwner != "mesh-delivery-builder" { t.Errorf("secrets-owner = %q", m.SecretsOwner) } - for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "defines-seats", "tools", "seat", "seats", "bus", "config"} { + // 1945: the builder's tools, served by its tools bundle, and the event it says when it asks an ask again. + if want := []string{"builder_ask_again", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) { + t.Errorf("tools = %v, want %v", m.Tools, want) + } + if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) { + t.Errorf("emits = %v, want %v", m.Emits, want) + } + for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the builder has none", k) } @@ -127,7 +137,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1 procs = append(procs, r) } if r["type"] == "tool" || r["type"] == "seat" { - t.Errorf("resource %v: the builder serves no tool and holds no seat", r) + t.Errorf("resource %v: the builder's tools are its bundle's, and it holds no seat", r) } } if len(procs) != 1 { @@ -149,8 +159,13 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1 if _, bus := p["bus"]; bus { t.Errorf("the process declares a bus: it runs on its own broker credential") } - if len(m.Build.Artifacts) != 1 { - t.Fatalf("artifacts = %+v, want the program alone", m.Build.Artifacts) + if len(m.Build.Artifacts) != 2 { + t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts) + } + if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-builder-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-builder-tools"}) { + t.Errorf("tools artifact = %+v", a) // 1945 + } else if st, err := os.Stat(a.From); err != nil || !st.IsDir() { + t.Errorf("the tools' source %s: %v", a.From, err) } a := m.Build.Artifacts[0] if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-builder" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index e15c633..7d62e68 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -2,6 +2,13 @@ "module": "mesh-delivery-builder", "version": "1", "slug": "builder", + "tools": [ + "builder_ask_again", + "builder_stuck" + ], + "emits": [ + "build.asked-again" + ], "consumes": [ "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", @@ -12,7 +19,8 @@ ], "state": [ { "name": "builds", "history": 5 }, - { "name": "instances", "history": 1 } + { "name": "instances", "history": 1 }, + { "name": "reasks", "history": 1 } ], "reads": [ "mesh-build-go.steps", @@ -78,6 +86,17 @@ "system": "arch", "from": "cmd/mesh-delivery-builder", "binary": "mesh-delivery-builder" + }, + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-delivery-builder-tools", + "binary": "mesh-delivery-builder-tools", + "loads": [ + "mesh-delivery-builder-tools" + ] } ] } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 297cfce..c46ec4d 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -152,17 +152,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - bucket := func(name string) (jetstream.KeyValue, error) { - kv, err := js.KeyValue(ctx, name) - if err != nil { - return nil, fmt.Errorf("the bucket %s: %w", name, err) - } - return kv, nil - } + // Its own states, the candidates it checks, and the modules and manifests that say which build modules judge a repository: + // a check without one of them would be wrong or unwritten, so each stops the start, naming its grant (issue 1945). buckets := map[string]jetstream.KeyValue{} for _, name := range []string{delivery.CheckerVerdictsBucket, delivery.CheckerKeptBucket, delivery.CheckerOwedBucket, delivery.PreviewsBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.ManifestsBucket, check.TriesBucket} { - if buckets[name], err = bucket(name); err != nil { + if buckets[name], err = process.EssentialBucket(ctx, js, "checker", name, "judge or keep a check"); err != nil { return err } } @@ -212,12 +207,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line // says nothing of the builds, until the checker starts again. var builds check.BuildStates - switch kv, err := js.KeyValue(ctx, delivery.BuilderBuildsBucket); { - case errors.Is(err, jetstream.ErrBucketNotFound): - logger.Printf("the bucket %s does not exist yet: the line says nothing of the builds until the checker starts again", delivery.BuilderBuildsBucket) - case err != nil: - return fmt.Errorf("the bucket %s: %w", delivery.BuilderBuildsBucket, err) - default: + if kv := process.OptionalBucket(ctx, js, delivery.BuilderBuildsBucket, "the line says nothing of the builds until the checker starts again", logger.Printf); kv != nil { builds = buildReader{kvStore{kv}} } // The assigner's compositions (ADR 0528 clause 5, issue 1297): their bucket is looked up at each check until it is @@ -290,47 +280,38 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // the assigner's own presence says whether a composition is being made (compositionWatch, ADR 0537 §4). A check is no // longer held for a node: its asks wait in the build seats' streams, which redeliver a lost node's ask (ADR 0525 §6, // issue 1912), so the node-engine's have is not watched here. - switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); { - case errors.Is(err, jetstream.ErrBucketNotFound): - logger.Printf("the bucket %s does not exist yet: no node counts as online until the checker starts again", delivery.PresenceBucket) - case err != nil: - return fmt.Errorf("the bucket %s: %w", delivery.PresenceBucket, err) - default: - pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) }) - if err != nil { - return err + if kv := process.OptionalBucket(ctx, js, delivery.PresenceBucket, "no node counts as online until the checker starts again", logger.Printf); kv != nil { + if pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) }); err != nil { + process.NotWatched(logger.Printf, delivery.PresenceBucket, err, "no node counts as online until the checker starts again") + } else { + go func() { + errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) { + if presence.Put(e.Key(), e.Value(), !opOf(e)) { + ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard + } + }) + }() } - go func() { - errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) { - if presence.Put(e.Key(), e.Value(), !opOf(e)) { - ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard - } - }) - }() } // The gate's source says the trunk head of each repository, which a pull request is merged onto. Where it is not there // yet no trunk is known, so nothing is checked. - switch kv, err := js.KeyValue(ctx, delivery.SourceBucket); { - case errors.Is(err, jetstream.ErrBucketNotFound): - logger.Printf("the bucket %s does not exist yet: no trunk is known until the checker starts again", delivery.SourceBucket) - case err != nil: - return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err) - default: + if kv := process.OptionalBucket(ctx, js, delivery.SourceBucket, "no trunk is known, so nothing is checked until the checker starts again", logger.Printf); kv != nil { sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { if e.Key() == delivery.SourceMainKey { trunks.Put(e.Value(), !opOf(e)) } }) if err != nil { - return err + process.NotWatched(logger.Printf, delivery.SourceBucket, err, "no trunk is known, so nothing is checked until the checker starts again") + } else { + go func() { + errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { + if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) { + ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again + } + }) + }() } - go func() { - errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { - if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) { - ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again - } - }) - }() } // The modules say which repository holds a TypeScript module, so the Node build module judges it too. mw, err := settle(ctx, buckets[delivery.ModulesBucket], func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) }) diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing.go index 3d6decf..9a06cec 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing.go @@ -193,7 +193,9 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) } w, err := settle(ctx, kv, each) if err != nil { - return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err) + // Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read. + logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err) + return errs, nil } if gate.Settled() { changed(ctx) diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go index 28fccc4..0715aea 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/standing_test.go @@ -3,11 +3,19 @@ package main // The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940). import ( + "bytes" + "context" + "errors" "fmt" + "log" "strings" "testing" "time" + "git.novox.be/novox/mesh-sdk/go/process" + + "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) @@ -89,6 +97,51 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) { if n := len(set.Entries("typescript")); n != 2 { t.Fatalf("novox and shanks are left; got %d entries", n) } + + // 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every + // toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without + // is an error naming the bucket and its grant. + var logs bytes.Buffer + logger := log.New(&logs, "", 0) + refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")} + if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil { + t.Errorf("a refused watch of the assignments stopped the start: %v", err) + } + if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil { + t.Errorf("a refused toolchains bucket stopped the start: %v", err) + } + if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") { + t.Errorf("the refusals were not said: %q", logs.String()) + } + if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil || + !strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") { + t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err) + } +} + +// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err. +type refusingBus struct { + refuseOpen bool + err error +} + +func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { + if r.refuseOpen { + return nil, r.err + } + return refusedWatch{err: r.err}, nil +} + +type refusedWatch struct { + jetstream.KeyValue + err error +} + +func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + return nil, r.err +} +func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + return nil, r.err } // A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go index bc848ca..99c3087 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go @@ -8,6 +8,8 @@ import ( "strings" "sync" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -63,11 +65,14 @@ func (w *toolchainWatch) ensure(module string) error { w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module) return nil case err != nil: - return fmt.Errorf("the bucket %s: %w", name, err) + // Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report. + w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module) + return nil } watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) }) if err != nil { - return err + process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module)) + return nil } w.watched[module] = true go func() { diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index e753fa3..5c4d652 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-checker go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum index 744af4c..4e82aa0 100644 --- a/modules/mesh-delivery-checker/go.sum +++ b/modules/mesh-delivery-checker/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 h1:FAiFY2KBKEoZSH49OZTt7/zRBIQxyl5g/KhYrpggbN4= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index a11a3b2..e069a6c 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -22,6 +22,8 @@ import ( "syscall" "time" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -188,18 +190,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro call := callerOn(conn, cred.User) // No forge event is heard while the open pull requests are read and said (ADR 0443 §5): events and syncs take turns. var turn sync.Mutex - kv, err := js.KeyValue(ctx, headsBucket) + kv, err := process.EssentialBucket(ctx, js, "bridge", headsBucket, "keep the heads it wrote on the forge stream") if err != nil { - return fmt.Errorf("the bucket %s: %w", headsBucket, err) + return err } heads := kvHeads{kv} // Everything written on FORGE, an event's and a sync's, keeps the head in the bucket. rec := &recorder{pub: js, heads: heads, logger: logger} said := func(ctx context.Context) error { return syncOpen(ctx, call, rec, heads, &turn, logger) } // What the gate asks of the forge (ADR 0437 §8): its owed acts and its merges, each performed once per revision. - performedKV, err := js.KeyValue(ctx, performedBucket) + performedKV, err := process.EssentialBucket(ctx, js, "bridge", performedBucket, "perform an act of the gate only once") if err != nil { - return fmt.Errorf("the bucket %s: %w", performedBucket, err) + return err } perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger} watching := make(chan error, 3) @@ -207,16 +209,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro bucket string react func(ctx context.Context, key string, rev uint64, value []byte) }{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, perf.merges}} { - kv, err := js.KeyValue(ctx, w.bucket) - if errors.Is(err, jetstream.ErrBucketNotFound) { - // The gate's buckets come with the gate; the bridge does not wait for them (no timer), and is started again - // by its delivery when the gate arrives. - logger.Printf("the bucket %s does not exist yet: nothing of it is performed until the bridge starts again", w.bucket) + // The gate's buckets come with the gate and its grant; the bridge does not wait for them (no timer), and is started + // again by its delivery when the gate arrives. A bucket refused or not answered is the same (issue 1945). + kv := process.OptionalBucket(ctx, js, w.bucket, "nothing of it is performed until the bridge starts again", logger.Printf) + if kv == nil { continue } - if err != nil { - return fmt.Errorf("the bucket %s: %w", w.bucket, err) - } go func() { watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) { hctx, cancel := context.WithTimeout(ctx, syncDeadline) @@ -231,7 +229,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro go onReconnect(ctx, again, said, logger) cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module) if err != nil { - return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, eventsStream, err) + // Essential: the forge's events are what the bridge carries. + return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the bridge hears no forge event without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, eventsStream, err) } cc, err := cons.Consume(func(m jetstream.Msg) { hear(&turn, func() { diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go index c4ab010..9a16a64 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "log" "slices" "strings" @@ -12,6 +13,8 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -66,6 +69,16 @@ func body(m map[string]any) []byte { } func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit part; the bus is the integration test) + t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) { + refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x") + var said []string + if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") { + t.Errorf("an optional bucket refused: %v, said %v", kv, said) + } + if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "bridge", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") { + t.Errorf("an essential bucket refused: %v, want its grant named", err) + } + }) var logs bytes.Buffer m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{} handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0)) @@ -230,3 +243,8 @@ func someFiles(_, _, path, _ string) ([]byte, error) { } return nil, bridge.ErrNotFound } + +// refusingBus refuses every bucket with err (issue 1945). +type refusingBus struct{ err error } + +func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err } diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index 669a04d..6f902ec 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-forge-bridge/go.sum b/modules/mesh-delivery-forge-bridge/go.sum index db1ea18..bdc8b83 100644 --- a/modules/mesh-delivery-forge-bridge/go.sum +++ b/modules/mesh-delivery-forge-bridge/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index fbf6ff3..a4a515f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -240,29 +240,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - bucket := func(name string) (jetstream.KeyValue, error) { - kv, err := js.KeyValue(ctx, name) - if err != nil { - return nil, fmt.Errorf("the bucket %s: %w", name, err) - } - return kv, nil - } - candidates, err := bucket(delivery.CandidatesBucket) + candidates, err := process.EssentialBucket(ctx, js, "indexer", delivery.CandidatesBucket, "write a candidate's fingerprints") if err != nil { return err } - modules, err := bucket(delivery.ModulesBucket) + modules, err := process.EssentialBucket(ctx, js, "indexer", delivery.ModulesBucket, "write a module's fingerprint") if err != nil { return err } - manifests, err := bucket(delivery.ManifestsBucket) - if err != nil { - return err - } - builds, err := bucket(delivery.BuildsBucket) + manifests, err := process.EssentialBucket(ctx, js, "indexer", delivery.ManifestsBucket, "keep a module's manifest") if err != nil { return err } + // The controller's build records only name the directory of a module whose key names none (fillPaths): without them + // such a module waits for its path, every other is indexed (issue 1945). + builds := process.OptionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf) known := index.NewBuilds() // The controller's build records name the directory of a module whose key names none (fillPaths); no build of any // module moves a fingerprint since no fingerprint takes the builder (ADR 0530 §1). @@ -273,12 +265,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro known.Delete(e.Key()) } } - w, err := settle(ctx, builds, each) - if err != nil { - return err - } errs := make(chan error, 1) - go func() { errs <- follow(ctx, w, each) }() + if builds != nil { + if w, err := settle(ctx, builds, each); err != nil { + process.NotWatched(logger.Printf, delivery.BuildsBucket, err, "a module whose key names no directory is not given one until the indexer starts again") + } else { + go func() { errs <- follow(ctx, w, each) }() + } + } // The indexer asks the forge for the objects of a manifest's named paths as its own account (ADR 0499 clause 3). account, err := process.ReadAccount(path) @@ -296,22 +290,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does // not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate. sourceErrs := make(chan error, 1) - switch source, err := js.KeyValue(ctx, delivery.SourceBucket); { - case errors.Is(err, jetstream.ErrBucketNotFound): - logger.Printf("the bucket %s does not exist yet: no source revision is read until the indexer starts again", delivery.SourceBucket) - case err != nil: - return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err) - default: + if source := process.OptionalBucket(ctx, js, delivery.SourceBucket, "no source revision is read until the indexer starts again", logger.Printf); source != nil { each := func(e jetstream.KeyValueEntry) { if opOf(e) { applySource(ctx, ix, e.Key(), e.Revision(), e.Value(), logger) } } - sw, err := settle(ctx, source, each) - if err != nil { - return err + if sw, err := settle(ctx, source, each); err != nil { + process.NotWatched(logger.Printf, delivery.SourceBucket, err, "no source revision is read until the indexer starts again") + } else { + go func() { sourceErrs <- follow(ctx, sw, each) }() } - go func() { sourceErrs <- follow(ctx, sw, each) }() } // Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's // assignments are settled first, so the toolchains settle through the gate already knowing them. @@ -381,7 +370,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } cons, err := js.Consumer(ctx, forgeStream, cred.Node+"_"+cred.Module) if err != nil { - return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, forgeStream, err) + // Essential: the forge's merges are what the indexer indexes. + return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the indexer hears no merge without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, forgeStream, err) } cc, err := cons.Consume(func(m jetstream.Msg) { handle(ctx, m, ix, logger) }, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) })) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go index c42221c..3b95a25 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go @@ -3,7 +3,7 @@ package main import ( "context" "errors" - "fmt" + "git.novox.be/novox/mesh-sdk/go/process" "log" "strings" "sync" @@ -55,11 +55,14 @@ func (w *previewWatch) ensure() error { w.logger.Printf("the bucket %s does not exist yet: it is watched when the checker says it wrote a preview", delivery.PreviewsBucket) return nil case err != nil: - return fmt.Errorf("the bucket %s: %w", delivery.PreviewsBucket, err) + // Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the checker's next preview. + w.logger.Printf("the bucket %s cannot be read (%v): it is watched when the checker says it wrote a preview", delivery.PreviewsBucket, err) + return nil } watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutPreview(e.Key(), e.Value(), !opOf(e)) }) if err != nil { - return err + process.NotWatched(w.logger.Printf, delivery.PreviewsBucket, err, "it is watched when the checker says it wrote a preview") + return nil } w.watched = true go func() { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing.go index 3d6decf..9a06cec 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/standing.go @@ -193,7 +193,9 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) } w, err := settle(ctx, kv, each) if err != nil { - return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err) + // Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read. + logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err) + return errs, nil } if gate.Settled() { changed(ctx) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains.go index 30892ae..92aafa4 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains.go @@ -10,6 +10,8 @@ import ( "strings" "sync" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -73,11 +75,14 @@ func (w *toolchainWatch) ensure(module string) error { w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module) return nil case err != nil: - return fmt.Errorf("the bucket %s: %w", name, err) + // Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report. + w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module) + return nil } watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.sink.put(module, e.Key(), e.Value(), !opOf(e)) }) if err != nil { - return err + process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module)) + return nil } w.watched[module] = true go func() { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains_test.go index e26bf76..7427efb 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/toolchains_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "errors" "log" "strings" "sync" @@ -21,6 +22,7 @@ type laterBuckets struct { mu sync.Mutex appears map[string]jetstream.KeyValue // buckets that exist now asked map[string]int + refuse error // answered for a bucket that does not exist, when set: a bus that refuses it (issue 1945) } func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) { @@ -30,6 +32,9 @@ func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyVa if kv, ok := l.appears[name]; ok { return kv, nil } + if l.refuse != nil { + return nil, l.refuse + } return nil, jetstream.ErrBucketNotFound } @@ -100,6 +105,12 @@ func TestABuildModulesBucketThatAppearsLaterIsWatchedWhenItsEventArrives(t *test recompute: func(context.Context) error { mu.Lock(); recomputes++; mu.Unlock(); return nil }, } var logs bytes.Buffer + // 1945: a bus that refuses both buckets, or does not answer, leaves them unwatched, said, and the indexer running. + refusing := &laterBuckets{appears: map[string]jetstream.KeyValue{}, asked: map[string]int{}, refuse: errors.New("context deadline exceeded")} + if err := newToolchainWatch(ctx, refusing, sink, log.New(&logs, "", 0)).start(); err != nil || !strings.Contains(logs.String(), "cannot be read") { + t.Fatalf("a refused toolchains bucket: start %v, log %q; want it said and the start going on", err, logs.String()) + } + logs.Reset() w := newToolchainWatch(ctx, buckets, sink, log.New(&logs, "", 0)) if err := w.start(); err != nil { // neither bucket exists: nothing is watched, nothing is written, nobody waits t.Fatal(err) diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 0aaff15..e8728ce 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 2a3e9f3..8e94e90 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go index f9b9142..d73893d 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go @@ -22,6 +22,8 @@ import ( "strings" "syscall" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -119,12 +121,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return nil } if err != nil { - return fmt.Errorf("the bucket %s: %w", delivery.AsksBucket, err) + // Essential: the gate's asks are what the teller tells (issue 1945: the error names the grant). + return process.EssentialErr("teller", delivery.AsksBucket, "tell any of the gate's asks", err) } + // The answers it writes and what it asked: its own work, so each stops the start, naming its grant. var kvs [2]jetstream.KeyValue for i, name := range []string{delivery.AnswersBucket, askedBucket} { - if kvs[i], err = js.KeyValue(ctx, name); err != nil { - return fmt.Errorf("the bucket %s: %w", name, err) + if kvs[i], err = process.EssentialBucket(ctx, js, "teller", name, "write an answer or remember what it asked"); err != nil { + return err } } state := kvStore{ctx: ctx, kv: kvs[1]} @@ -140,7 +144,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The view's answers to the assigner's asks (issue 1297 B10): both buckets are the controller's to create, and a bucket not // there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer). failed := make(chan error, 2) - assignerKV, viewKV := optionalBucket(ctx, js, delivery.AssignerAsksBucket, logger), optionalBucket(ctx, js, delivery.ViewAnswersBucket, logger) + assignerKV, viewKV := process.OptionalBucket(ctx, js, delivery.AssignerAsksBucket, teller, logger.Printf), process.OptionalBucket(ctx, js, delivery.ViewAnswersBucket, teller, logger.Printf) // The assigner's asks are raised on the operator channel whether or not the view's answers can be read (issue 1906). if viewKV != nil { t.View = kvGate{ctx: ctx, kv: viewKV} @@ -172,15 +176,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } } -// optionalBucket is a bucket the teller reads when it exists, and nil, said once, when it does not. -func optionalBucket(ctx context.Context, js jetstream.JetStream, name string, logger *log.Logger) jetstream.KeyValue { - kv, err := js.KeyValue(ctx, name) - if err != nil { - logger.Printf("the bucket %s cannot be read (%v): the asks and answers it holds are not carried until the teller is started again with it", name, err) - return nil - } - return kv -} +// teller is what an optional bucket of the teller leaves out. +const teller = "the asks and answers it holds are not carried until the teller is started again with it" // askedBucket is the teller's state bucket asked, `_`: what it asked of the router and which condition it // raised (manifest state entry asked, novox/hq ADR 0437 §4). diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go index 8986842..ae05ae0 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go @@ -1,12 +1,17 @@ package main import ( + "context" "encoding/json" + "errors" "fmt" "strings" "testing" "time" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go/jetstream" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/asker" "git.novox.be/novox/mesh-sdk/go/asks" @@ -236,6 +241,16 @@ func (b *bed) askedOf(subject string) []published { } func TestAnAsksKeyBecomesOneRouterAskAndDeletingItCancelsIt(t *testing.T) { // 1343/AC1 + t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) { + refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x") + var said []string + if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") { + t.Errorf("an optional bucket refused: %v, said %v", kv, said) + } + if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "teller", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") { + t.Errorf("an essential bucket refused: %v, want its grant named", err) + } + }) b := newBed(t) key, value, ga := gateAsk(t, twoHeads(), twoHeads()) b.gate[key] = value @@ -518,3 +533,8 @@ func TestAnAnswerToAnEarlierAskingDoesNotBlockTheNextAsking(t *testing.T) { // R t.Errorf("writes %d asks %d dropped %v", b.answers.writes, len(b.askedOf(asks.AskSubject(delivery.TellerModule))), b.dropped) } } + +// refusingBus refuses every bucket with err (issue 1945). +type refusingBus struct{ err error } + +func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err } diff --git a/modules/mesh-delivery-teller/go.mod b/modules/mesh-delivery-teller/go.mod index db3470d..0937a16 100644 --- a/modules/mesh-delivery-teller/go.mod +++ b/modules/mesh-delivery-teller/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-teller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-teller/go.sum b/modules/mesh-delivery-teller/go.sum index 97c314d..3baa2a5 100644 --- a/modules/mesh-delivery-teller/go.sum +++ b/modules/mesh-delivery-teller/go.sum @@ -6,6 +6,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv4 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index 3ffe7d1..d0675db 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -9,8 +9,9 @@ import ( "os" "strings" - stdio "git.novox.be/novox/mesh-sdk/go" "mesh-delivery-view/internal/view" + + stdio "git.novox.be/novox/mesh-sdk/go" ) // Tools are the read tools over a modules state. diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index f619da4..57994b4 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -22,6 +22,8 @@ import ( "syscall" "time" + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -222,23 +224,23 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - bucket := func(name string) (jetstream.KeyValue, error) { - kv, err := js.KeyValue(ctx, name) - if err != nil { - return nil, fmt.Errorf("the bucket %s: %w", name, err) - } - return kv, nil - } + // Its own presence, pulses and stuck rows are what it keeps: each stops the start, naming its grant. The assigner's wants + // and the node-engine's haves only make the pulses of a move: without one the watchman watches presence and the build + // seats' steps, and says that moves are not watched (issue 1945). var kvs struct{ presence, pulse, stuck, wants, haves jetstream.KeyValue } for _, b := range []struct { name string into *jetstream.KeyValue - }{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck}, - {delivery.AssignerWantsBucket, &kvs.wants}, {delivery.HaveBucket, &kvs.haves}} { - if *b.into, err = bucket(b.name); err != nil { + }{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck}} { + if *b.into, err = process.EssentialBucket(ctx, js, "watchman", b.name, "say who is online or what stopped moving"); err != nil { return err } } + kvs.wants = process.OptionalBucket(ctx, js, delivery.AssignerWantsBucket, "a want makes no pulse, so a move that stops is not seen, until the watchman starts again", logger.Printf) + kvs.haves = process.OptionalBucket(ctx, js, delivery.HaveBucket, "a want makes no pulse, so a move that stops is not seen, until the watchman starts again", logger.Printf) + if kvs.wants == nil || kvs.haves == nil { + kvs.wants, kvs.haves = nil, nil // a want is a pulse only against the have that ends it + } w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck}, time.Now) w.Logf(logger.Printf) @@ -283,13 +285,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } // What the node agents run, then the assigner's wants (issue 1906, ADR 0539 §19): a want is a pulse owed by its node's agent // until the have names its fingerprint, so the haves are read first and a want already applied makes no pulse. - haveSeen, err := settle(ctx, kvs.haves, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }) - if err != nil { - return err - } - wantSeen, err := settle(ctx, kvs.wants, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) }) - if err != nil { - return err + var haveSeen, wantSeen jetstream.KeyWatcher + if kvs.haves != nil { + if haveSeen, err = settle(ctx, kvs.haves, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }); err != nil { + process.NotWatched(logger.Printf, delivery.HaveBucket, err, "moves are not watched until the watchman starts again") + } else if wantSeen, err = settle(ctx, kvs.wants, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) }); err != nil { + process.NotWatched(logger.Printf, delivery.AssignerWantsBucket, err, "moves are not watched until the watchman starts again") + haveSeen.Stop() + haveSeen = nil + } } // The build seats' steps: a step key `.` written is the ask taken (issue 1887 A8b). A seat whose build module // keeps no steps bucket (yet) is said and left; the asks of its stream are mirrored all the same. @@ -304,7 +308,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro lang := lang sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { stepTaken(w, lang, e) }) if err != nil { - return err + logger.Printf("the steps of %s cannot be watched (%v): its asks are mirrored and never seen taken", module, err) + continue } taken[lang] = sw } @@ -348,12 +353,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro go func() { errs <- follow(ctx, pulses, func(e jetstream.KeyValueEntry) { w.Pulse(e.Key(), opOf(e), e.Created()) }) }() - go func() { - errs <- follow(ctx, haveSeen, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }) - }() - go func() { - errs <- follow(ctx, wantSeen, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) }) - }() + if haveSeen != nil && wantSeen != nil { + go func() { + errs <- follow(ctx, haveSeen, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }) + }() + go func() { + errs <- follow(ctx, wantSeen, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) }) + }() + } for lang, sw := range taken { lang, sw := lang, sw go func() { errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { stepTaken(w, lang, e) }) }() diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go index 6a171c2..58fe0a7 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go @@ -2,11 +2,17 @@ package main import ( "context" + "errors" + "fmt" "io" "os" "path/filepath" "strings" "testing" + + "git.novox.be/novox/mesh-sdk/go/process" + + "github.com/nats-io/nats.go/jetstream" ) func TestCredentialIsTheBrokerFile(t *testing.T) { // 1135/AC6 @@ -27,8 +33,23 @@ func TestCredentialIsTheBrokerFile(t *testing.T) { // 1135/AC6 } func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6 + t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) { + refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x") + var said []string + if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") { + t.Errorf("an optional bucket refused: %v, said %v", kv, said) + } + if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "watchman", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") { + t.Errorf("an essential bucket refused: %v, want its grant named", err) + } + }) err := run(context.Background(), func(string) string { return "" }, io.Discard) if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") { t.Fatalf("run = %v, want a refusal naming MESH_BROKER_FILE", err) } } + +// refusingBus refuses every bucket with err (issue 1945). +type refusingBus struct{ err error } + +func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err } diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index d343e0e..b154f77 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-watchman go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-watchman/go.sum b/modules/mesh-delivery-watchman/go.sum index bfba8a6..3c78612 100644 --- a/modules/mesh-delivery-watchman/go.sum +++ b/modules/mesh-delivery-watchman/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=