From 28cb212030a6110dde77a430257a4bb7d07e3728 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 22:47:35 +0100 Subject: [PATCH 01/10] BL16: the builder puts its asks on accept..builder, and a step key neither form parses is named in builder_stuck's journal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The delivery SDK's buildstep gains AskerBuilder, AskSubjectBy and ParseAskSubjectBy (an ask's subject names its asker as the last token, main's form names none; ParseAskSubject reads both). The builder's put and its held-ask reads use the builder's subject. Not built here: §543's refusal (the build modules'), the grants (grants part). Builds: design 47 G §481, §484, §543, §600 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-delivery-builder/buildrun/asks.go | 5 ++- .../buildrun/asks_test.go | 18 ++++++++- .../cmd/mesh-delivery-builder/held.go | 4 +- .../internal/build/run_test.go | 21 ++++++++++ .../internal/build/stuck.go | 10 ++++- sdk/buildstep/ask.go | 39 +++++++++++++++---- sdk/buildstep/ask_test.go | 26 +++++++++++++ 7 files changed, 109 insertions(+), 14 deletions(-) diff --git a/modules/mesh-delivery-builder/buildrun/asks.go b/modules/mesh-delivery-builder/buildrun/asks.go index d9a0c35b..bbc793fb 100644 --- a/modules/mesh-delivery-builder/buildrun/asks.go +++ b/modules/mesh-delivery-builder/buildrun/asks.go @@ -102,7 +102,8 @@ func (k Asker) dropped(a buildstep.StepAsk, why string) { // PutTry is how long one publish of an ask waits for its stream's answer before it is made again (Put). var PutTry = 10 * time.Second -// Put puts a on its stream: the seat's accept subject of its language and kind. The put is a kept call (novox/hq ADR 0668 +// Put puts a on its stream: the seat's accept subject of its language and kind, under the builder's name as asker (design 47 G +// §481, §600: `accept..builder`). The put is a kept call (novox/hq ADR 0668 // §1, §2; issue 1966, issue 1748 note 27): it carries the ask's id as its message id, so the stream takes it once, and it // is made again through every transport fault (a deadline, no responders) within busretry.Bound, each try bounded by // PutTry, until ctx ends. A put that still meets one is an error opening with buildstep.TransportFaultWords, so the key it @@ -112,7 +113,7 @@ func (k Asker) Put(ctx context.Context, a buildstep.StepAsk) error { if err != nil { return err } - subject, err := buildstep.AskSubject(a.Language, a.Kind) + subject, err := buildstep.AskSubjectBy(a.Language, a.Kind, buildstep.AskerBuilder) if err != nil { return err } diff --git a/modules/mesh-delivery-builder/buildrun/asks_test.go b/modules/mesh-delivery-builder/buildrun/asks_test.go index a460f924..ed127a56 100644 --- a/modules/mesh-delivery-builder/buildrun/asks_test.go +++ b/modules/mesh-delivery-builder/buildrun/asks_test.go @@ -30,7 +30,8 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { f := &FakeAsks{Answer: func(a buildstep.StepAsk) buildstep.Step { return buildstep.Step{ID: a.ID, Node: "ace", State: buildstep.StepPassed, Result: []byte(`{"artifacts":[]}`)} }} - k := Asker{Publisher: f, Waiter: f} + subjects := &subjectsOf{inner: f} + k := Asker{Publisher: subjects, Waiter: f} var seen, also []buildstep.Step // issue 1890: a waiter that asks to see each value of the step sees it // Issue 1921: a second seen on the same context (a check's begin bound inside a try) is added, never in place of the first. both := WithStepSeen(WithStepSeen(context.Background(), func(s buildstep.Step) { seen = append(seen, s) }), @@ -39,6 +40,10 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { if err != nil || step.Node != "ace" || len(f.Asks()) != 1 || len(seen) != 1 || seen[0].Node != "ace" || len(also) != 1 { t.Fatalf("step %+v, %v, asks %d, seen %+v and %+v", step, err, len(f.Asks()), seen, also) } + // Design 47 G §481, §600: the ask is put under the builder's name, accept..builder. + if len(subjects.seen) != 1 || subjects.seen[0] != "mesh.seat.node-build-go.accept."+goAsk("x").Kind+".builder" { + t.Fatalf("the ask was put on %v", subjects.seen) + } f.Answer = func(a buildstep.StepAsk) buildstep.Step { return buildstep.Step{ID: a.ID, Node: "shanks", State: buildstep.StepRefused, Reason: "go 1.27 installed"} } @@ -381,3 +386,14 @@ func (p *faultyPuts) PublishAsk(ctx context.Context, subject, id string, data [] } return p.inner.PublishAsk(ctx, subject, id, data) } + +// subjectsOf records the subject each ask is published on. +type subjectsOf struct { + inner AskPublisher + seen []string +} + +func (p *subjectsOf) PublishAsk(ctx context.Context, subject, id string, data []byte) error { + p.seen = append(p.seen, subject) + return p.inner.PublishAsk(ctx, subject, id, data) +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go index acdbd9fe..f8cb1e46 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go @@ -115,7 +115,7 @@ func (h heldOnTheBus) seen(ctx context.Context, a buildstep.StepAsk, later bool) // stream is the build seat's stream of language, its name and the subject of kind on it. func (h heldOnTheBus) stream(ctx context.Context, language, kind string) (jetstream.Stream, string, string, error) { - subject, err := buildstep.AskSubject(language, kind) + subject, err := buildstep.AskSubjectBy(language, kind, buildstep.AskerBuilder) if err != nil { return nil, "", "", err } @@ -286,7 +286,7 @@ func (h heldOnTheBus) Queue(ctx context.Context, language string, limit int) (bu return q, err } for _, kind := range buildstep.AskKinds { - subject, _ := buildstep.AskSubject(language, kind) + subject, _ := buildstep.AskSubjectBy(language, kind, buildstep.AskerBuilder) var ahead []build.Queued if err := onSubject(ctx, stream, subject, func(m build.Queued) bool { ahead = append(ahead, m) diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 1c471d37..ce3d66f3 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -1328,6 +1328,27 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { if maxFlight.Load() < 2 { t.Errorf("the values were read one at a time") } + // Design 47 G §484: a step key neither form parses is written in the journal naming it. + var said []string + steps := StateRead{Keys: func() ([]string, error) { + return []string{"ace.mvl0frv18d1957", "builder.ace.mvl0frv18d1958", "a.b.c.d"}, nil + }} + if _, err := (StuckReader{Builds: slow, ReAsks: state(map[string]string{}), Steps: map[string]StateRead{"mesh-build-go": steps}, Now: func() time.Time { return at }, + Logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}).Stuck("mod10"); err != nil { + t.Fatal(err) + } + n := 0 + for _, l := range said { + if strings.Contains(l, `"a.b.c.d"`) { + n++ + } + if strings.Contains(l, "ace.mvl0frv18d1957") || strings.Contains(l, "builder.ace.mvl0frv18d1958") { + t.Errorf("a key of one of the two forms is said not to parse: %s", l) + } + } + if n != 1 { + t.Errorf("the key of neither form is named %d times in %q", n, said) + } }) } diff --git a/modules/mesh-delivery-builder/internal/build/stuck.go b/modules/mesh-delivery-builder/internal/build/stuck.go index 04f3fd1a..560bce85 100644 --- a/modules/mesh-delivery-builder/internal/build/stuck.go +++ b/modules/mesh-delivery-builder/internal/build/stuck.go @@ -325,9 +325,15 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) { } byID := make(map[string]string, len(ks)) for _, k := range ks { - if _, _, id, err := buildstep.ParseStepKey(k); err == nil { - byID[id] = k + _, _, id, err := buildstep.ParseStepKey(k) + if err != nil { + // A key neither form parses is named, never passed over in silence (design 47 G §484). + if r.Logf != nil { + r.Logf("builder_stuck: the key %q of the steps of %s is a step key of neither form, .. or .: %v", k, bm, err) + } + continue } + byID[id] = k } stepKeys[bm] = byID } diff --git a/sdk/buildstep/ask.go b/sdk/buildstep/ask.go index 55a63241..6281a77d 100644 --- a/sdk/buildstep/ask.go +++ b/sdk/buildstep/ask.go @@ -137,7 +137,8 @@ func BuildModuleOf(language string) (string, error) { return "mesh-build-" + language, nil } -// AskSubject is the subject a build seat accepts a kind of ask on: the seat's accept subject. +// AskSubject is the subject a build seat accepts a kind of ask on: the seat's accept subject of main's form, which names no +// asker (design 47 G §600, §483). func AskSubject(language, kind string) (string, error) { seat, err := BuildSeat(language) if err != nil { @@ -149,20 +150,44 @@ func AskSubject(language, kind string) (string, error) { return "mesh.seat." + seat + ".accept." + kind, nil } -// ParseAskSubject is the inverse of AskSubject. +// AskerBuilder is the builder's name as an asker: the last token of the subject its asks are put on and the first of its +// step keys (design 47 G §600, §481). +const AskerBuilder = "builder" + +// AskSubjectBy is the subject a build seat accepts a queue's ask of asker on: `mesh.seat..accept..`, the +// asker's name the last token (design 47 G §481, §600). An empty asker is main's subject (AskSubject). +func AskSubjectBy(language, kind, asker string) (string, error) { + subject, err := AskSubject(language, kind) + if err != nil || asker == "" { + return subject, err + } + if !askerOK.MatchString(asker) { + return "", fmt.Errorf("buildstep: %q is not an asker's name", asker) + } + return subject + "." + asker, nil +} + +// ParseAskSubject is the inverse of AskSubject and of AskSubjectBy: the language and the queue of either form. func ParseAskSubject(subject string) (language, kind string, err error) { + language, kind, _, err = ParseAskSubjectBy(subject) + return language, kind, err +} + +// ParseAskSubjectBy is the inverse of AskSubjectBy: the language, the queue and the asker, which is empty for main's form. +func ParseAskSubjectBy(subject string) (language, kind, asker string, err error) { rest, ok := strings.CutPrefix(subject, "mesh.seat.node-build-") if !ok { - return "", "", fmt.Errorf("buildstep: %q is not a build seat's accept subject", subject) + return "", "", "", fmt.Errorf("buildstep: %q is not a build seat's accept subject", subject) } language, kind, ok = strings.Cut(rest, ".accept.") if !ok { - return "", "", fmt.Errorf("buildstep: %q is not a build seat's accept subject", subject) + return "", "", "", fmt.Errorf("buildstep: %q is not a build seat's accept subject", subject) } - if want, err := AskSubject(language, kind); err != nil || want != subject { - return "", "", fmt.Errorf("buildstep: %q is not a build seat's accept subject: %v", subject, err) + kind, asker, _ = strings.Cut(kind, ".") + if want, err := AskSubjectBy(language, kind, asker); err != nil || want != subject { + return "", "", "", fmt.Errorf("buildstep: %q is not a build seat's accept subject: %v", subject, err) } - return language, kind, nil + return language, kind, asker, nil } // AskStream is the work-queue stream of a language's build seat. diff --git a/sdk/buildstep/ask_test.go b/sdk/buildstep/ask_test.go index b394f66f..88558bee 100644 --- a/sdk/buildstep/ask_test.go +++ b/sdk/buildstep/ask_test.go @@ -594,3 +594,29 @@ func TestFetchBuildsResolvedAndToolchainDigestAreRead(t *testing.T) { t.Errorf("toolchain digest: %+v", tc) } } + +// §481, §600: a queue's ask of an asker is put on accept.., and both forms parse. +func TestAskSubjectBy(t *testing.T) { + got, err := AskSubjectBy(AskGo, AskDelivery, AskerBuilder) + if err != nil || got != "mesh.seat.node-build-go.accept.delivery.builder" { + t.Fatalf("%q %v", got, err) + } + for _, c := range []struct{ subject, language, kind, asker string }{ + {got, "go", "delivery", "builder"}, + {"mesh.seat.node-build-node.accept.check", "node", "check", ""}, + } { + l, k, a, err := ParseAskSubjectBy(c.subject) + if err != nil || l != c.language || k != c.kind || a != c.asker { + t.Errorf("%s: %q %q %q %v", c.subject, l, k, a, err) + } + if l, k, err := ParseAskSubject(c.subject); err != nil || l != c.language || k != c.kind { + t.Errorf("%s: ParseAskSubject %q %q %v", c.subject, l, k, err) + } + } + if _, err := AskSubjectBy(AskGo, AskDelivery, "Bad.Name"); err == nil { + t.Error("a bad asker has a subject") + } + if _, _, _, err := ParseAskSubjectBy("mesh.seat.node-build-go.accept.delivery.builder.extra"); err == nil { + t.Error("a subject of four tokens after accept parses") + } +} From 661b735060a7c1e4db3fb35e61494b8dc45053c5 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 22:47:54 +0100 Subject: [PATCH 02/10] D-29 (builder half, waits): the builder's old state waits declares max-mib 4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit publish-queue 4 and acted 1 come with BL12, which makes the sections. Builds: design 47 G §695, §672 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/mesh-delivery-builder/manifest_test.go | 3 ++- modules/mesh-delivery-builder/module.json | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 79789582..835fd25d 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -23,6 +23,7 @@ type state struct { Name string `json:"name"` History int `json:"history,omitempty"` KeyTTL bool `json:"key-ttl,omitempty"` + MaxMiB int `json:"max-mib,omitempty"` } type artifact struct { @@ -111,7 +112,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if m.Module != "mesh-delivery-builder" { t.Errorf("module = %q", m.Module) } - if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1}, {Name: "reserved", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory + if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory t.Errorf("state = %+v, want %+v", m.State, want) } // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index ac053db9..e43d5995 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -110,7 +110,7 @@ { "name": "bounds", "history": 1, "key-ttl": true }, { "name": "queues", "history": 1 }, { "name": "toolchains-heard", "history": 1 }, - { "name": "waits", "history": 1 }, + { "name": "waits", "history": 1, "max-mib": 4 }, { "name": "reserved", "history": 1 }, { "name": "reasks", "history": 1 }, { "name": "versions", "history": 1 }, From 5906cecf19a48d1fb5df797c186230484fa9515e Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 22:59:28 +0100 Subject: [PATCH 03/10] BL16 (integration bed): the test bed's build seat workers take both forms of an accept subject, and the stale ask is put as the builder's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §600, §481 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-builder/integration_test.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index 7feaa294..19619406 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -165,6 +165,18 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) { if err := runner.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil { t.Fatal(err) } + // Design 47 G §600: each queue's worker takes an ask in both forms, main's and the builder's (accept..builder); the + // build modules' helper above filters main's alone, so the worker is declared again with both. + stream, _ := sdkstep.AskStream(language) + for _, kind := range sdkstep.AskKinds { + worker, _ := sdkstep.AskWorker(language, kind) + main, _ := sdkstep.AskSubject(language, kind) + byBuilder, _ := sdkstep.AskSubjectBy(language, kind, sdkstep.AskerBuilder) + if _, err := f.js.CreateOrUpdateConsumer(f.ctx, stream, jetstream.ConsumerConfig{Durable: worker, FilterSubjects: []string{main, byBuilder}, + AckPolicy: jetstream.AckExplicitPolicy, AckWait: 30 * time.Second, MaxDeliver: 5}); err != nil { + t.Fatal(err) + } + } put := func(key string, value any) (uint64, error) { raw, _ := value.(json.RawMessage) return kv.Put(f.ctx, key, raw) @@ -1068,7 +1080,7 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) { held := heldOnTheBus{b.js} goAsk := sdkstep.StepAsk{ID: v.Asks[sdkstep.AskGo], Language: sdkstep.AskGo, Kind: sdkstep.AskDelivery} stale := sdkstep.StepAsk{ID: "mvo53jcm70b5f7", Language: sdkstep.AskGo, Kind: sdkstep.AskDelivery} - subject, _ := sdkstep.AskSubject(sdkstep.AskGo, sdkstep.AskDelivery) + subject, _ := sdkstep.AskSubjectBy(sdkstep.AskGo, sdkstep.AskDelivery, sdkstep.AskerBuilder) if _, err := b.js.Publish(b.ctx, subject, []byte("{}"), jetstream.WithMsgID(stale.ID)); err != nil { t.Fatal(err) } From 60b1479e8ab64e0547bb3850ecbe80188a7c3095 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:03:11 +0100 Subject: [PATCH 04/10] A-71 (builder): toolchain-wait-minutes reaches the builder as a file the manifest declares, read at its start, never its environment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manifest gains the file resource toolchain-wait (content ${config:toolchain-wait-minutes}) and names its path in MESH_BUILDER_TOOLCHAIN_WAIT_FILE; main.go's toolchainWaitOf reads it. Builds: design 47 G §672, A-71 (G-20) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-builder/main.go | 37 ++++++++++++++---- .../toolchainwait_test.go | 39 +++++++++++++++++++ .../mesh-delivery-builder/manifest_test.go | 19 ++++++++- modules/mesh-delivery-builder/module.json | 10 ++++- 4 files changed, 95 insertions(+), 10 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchainwait_test.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index b873b4c2..407f642c 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -7,8 +7,10 @@ package main import ( "cmp" "context" + "errors" "fmt" "io" + "io/fs" "log" "maps" "os" @@ -200,13 +202,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro b.Numbers = build.VersionNumbers{N: numberer} // The builder's toolchain waits (design 47 G §672): its state waits, and wait-held and wait-ended on its stream waits; its // bound is its setting toolchain-wait-minutes. - waitBound := defaultToolchainWait - if v := strings.TrimSpace(getenv("MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES")); v != "" { - n, err := strconv.Atoi(v) - if err != nil || n < 1 { - return fmt.Errorf("MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES %q is not a number of minutes", v) - } - waitBound = time.Duration(n) * time.Minute + waitBound, err := toolchainWaitOf(getenv("MESH_BUILDER_TOOLCHAIN_WAIT_FILE")) + if err != nil { + return err } waitsBucket := process.AwaitBucket(ctx, js, conn, build.WaitsBucket, "the toolchain waits are kept in memory only, and builder_waits shows none", logger.Printf) b.Waits = newWaitsKept(waitsBucket, func(ctx context.Context, event, id string, body any) error { @@ -569,3 +567,28 @@ func (s *sourceHeads) head(_ context.Context, rev uint64, repository string) (st } return "", nil } + +// toolchainWaitOf reads the toolchain wait's setting, toolchain-wait-minutes, from the file the manifest declares with the +// content ${config:toolchain-wait-minutes} (design 47 G §672, A-71), at the builder's start: no file, or an empty one, is +// the default; a content that is no number of minutes is an error naming the file. +func toolchainWaitOf(path string) (time.Duration, error) { + if path == "" { + return defaultToolchainWait, nil + } + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return defaultToolchainWait, nil + } + if err != nil { + return 0, err + } + v := strings.TrimSpace(string(raw)) + if v == "" { + return defaultToolchainWait, nil + } + n, err := strconv.Atoi(v) + if err != nil || n < 1 { + return 0, fmt.Errorf("%s holds %q, which is not a number of minutes", path, v) + } + return time.Duration(n) * time.Minute, nil +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchainwait_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchainwait_test.go new file mode 100644 index 00000000..2d63b4ab --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchainwait_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +// A-71: the toolchain wait's setting is read at the start from the file the manifest declares, never from the environment. +func TestToolchainWaitIsReadFromItsFile(t *testing.T) { + dir := t.TempDir() + n := 0 + file := func(content string) string { + n++ + p := filepath.Join(dir, "w"+string(rune('a'+n))) + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return p + } + for _, c := range []struct { + name, path string + want time.Duration + bad bool + }{ + {"no file named", "", defaultToolchainWait, false}, + {"file absent", filepath.Join(dir, "absent"), defaultToolchainWait, false}, + {"empty", file(" \n"), defaultToolchainWait, false}, + {"minutes", file("15\n"), 15 * time.Minute, false}, + {"not a number", file("soon"), 0, true}, + {"zero", file("0"), 0, true}, + } { + got, err := toolchainWaitOf(c.path) + if (err != nil) != c.bad || got != c.want { + t.Errorf("%s: %v, %v; want %v, error %v", c.name, got, err, c.want, c.bad) + } + } +} diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 835fd25d..38debdc3 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -3,6 +3,7 @@ package manifest import ( "bytes" "encoding/json" + "fmt" "io/fs" "os" "path/filepath" @@ -221,8 +222,22 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1 t.Errorf("process = %v", p) } env, _ := p["env"].(map[string]any) - if len(env) != 2 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES"] != "${config:toolchain-wait-minutes}" { - t.Errorf("env = %v, want the broker file and the toolchain wait's setting (design 47 G §672)", p["env"]) + if len(env) != 2 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" { + t.Errorf("env = %v, want the broker file and the path of the toolchain wait's file, never the setting itself (design 47 G §672, A-71)", p["env"]) + } + for k, v := range env { + if strings.Contains(fmt.Sprint(v), "${config:") { + t.Errorf("env %s = %v: a setting reaches the process as a file, never its environment (A-71)", k, v) + } + } + found := false + for _, r := range m.Resources { + if r["type"] == "file" && r["path"] == "${dir:state}/toolchain-wait-minutes" && r["content"] == "${config:toolchain-wait-minutes}" { + found = true + } + } + if !found { + t.Errorf("no file resource with the content ${config:toolchain-wait-minutes}: A-71") } for _, r := range m.Resources { if r["id"] == "settings" { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index e43d5995..8e65cc39 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -165,6 +165,14 @@ "place": ".", "unknown": "program" }, + { + "id": "toolchain-wait", + "type": "file", + "path": "${dir:state}/toolchain-wait-minutes", + "owner": "mesh-delivery-builder", + "mode": "0600", + "content": "${config:toolchain-wait-minutes}" + }, { "id": "builder", "type": "process", @@ -175,7 +183,7 @@ ], "env": { "MESH_BROKER_FILE": "${dir:state}/broker", - "MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES": "${config:toolchain-wait-minutes}" + "MESH_BUILDER_TOOLCHAIN_WAIT_FILE": "${dir:state}/toolchain-wait-minutes" }, "user": "mesh-delivery-builder", "health": { From 5b6175714b24da9feb2a2b1fbe2bfe3373eef148 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:12:32 +0100 Subject: [PATCH 05/10] =?UTF-8?q?BL12=20/=20G-16=20=C2=A7588:=20the=20buil?= =?UTF-8?q?der=20publishes=20reserved=20versions=20again,=20on=20an=20even?= =?UTF-8?q?t=20by=20stream=20sequence=20and=20by=20builder=5Fpublish=5Faga?= =?UTF-8?q?in?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reservation keeps the fingerprint of the catalog entry it was made from (versions.Source, A-76). The state sections publish-queue (max-mib 4) and acted (max-mib 1); a consumer acting once per message of the node-engine seat's running and manifest-reads-changed (why returned) by its stream sequence, the running of every subject read at the start with lastPerSubject; one queue per holder sent one at a time as the registry seat's publish with the digest and no parts, the seat and the package name read from the entry (A-73, A-76); builder_publish_queue and the new builder_publish_again, a hand act. The old form goes: build.NewPublishAgain / ModePublishAgain, versions.Acts.PublishAgain and the Again* decision, and the integration step that refused it. Builds: design 47 G §588, §369, §695, A-73, A-76 (G-16, G-18, G-20) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-builder-tools/main.go | 12 +- .../mesh-delivery-builder-tools/republish.go | 65 ++ .../mesh-delivery-builder/integration_test.go | 8 - .../cmd/mesh-delivery-builder/main.go | 30 +- .../cmd/mesh-delivery-builder/reasks.go | 15 +- .../cmd/mesh-delivery-builder/republish.go | 146 ++++ .../mesh-delivery-builder/republish_test.go | 79 +++ .../cmd/mesh-delivery-builder/reserve.go | 5 + .../cmd/mesh-delivery-builder/toolchains.go | 12 +- .../cmd/mesh-delivery-builder/versions.go | 20 - .../internal/build/numbers.go | 2 +- .../internal/build/stuck.go | 35 +- .../internal/versions/again.go | 94 +-- .../internal/versions/republish.go | 646 ++++++++++++++++++ .../internal/versions/republish_test.go | 220 ++++++ .../internal/versions/reserve.go | 12 +- .../internal/versions/versions_test.go | 67 +- .../mesh-delivery-builder/manifest_test.go | 16 +- modules/mesh-delivery-builder/module.json | 29 +- 19 files changed, 1274 insertions(+), 239 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go create mode 100644 modules/mesh-delivery-builder/internal/versions/republish.go create mode 100644 modules/mesh-delivery-builder/internal/versions/republish_test.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index dea457ae..cc4309b2 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -54,17 +54,9 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q if a.Refused { return nil, errors.New(a.Answer) } - return map[string]any{"key": key, "name": a.Name, "do": a.Do, "version": a.Version, "answer": a.Answer}, nil + return map[string]any{"key": key, "name": a.Name, "answer": a.Answer}, nil } - nameFP := map[string]any{"type": "object", "required": []string{"name", "fp", "why"}, "properties": map[string]any{ - "name": map[string]any{"type": "string", "description": "the numbered name, /, as builder_versions lists it"}, - "fp": map[string]any{"type": "string", "description": "the fingerprint, src2:"}, - "why": map[string]any{"type": "string", "description": "why, in plain words"}}} return []stdio.Tool{ - {Name: versions.PublishAgainTool, Description: versions.PublishAgainDescription, Input: nameFP, - Run: func(args map[string]any) (any, error) { - return handAct(build.NewPublishAgain(str(args, "name"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())) - }}, {Name: versions.FloorAgainTool, Description: versions.FloorAgainDescription, Input: map[string]any{"type": "object", "required": []string{"name", "why"}, "properties": map[string]any{ "name": map[string]any{"type": "string", "description": "the numbered name, /"}, @@ -372,6 +364,8 @@ func main() { }} tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr) tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) + qs, as := stdio.State(versions.PublishQueueState), stdio.State(versions.ActedState) + tools = append(tools, republishTools(readerOf(qs), readerOf(as), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) ws := stdio.State(build.WaitsState) wr := versions.Reader{Keys: ws.Keys, Get: func(key string) (json.RawMessage, bool, error) { e, err := ws.Get(key) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go new file mode 100644 index 00000000..9b48df0f --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go @@ -0,0 +1,65 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-builder/internal/build" + "mesh-delivery-builder/internal/versions" +) + +// readerOf is a kept state as the versions package's reader. +func readerOf(s stdio.KeptState) versions.Reader { + return versions.Reader{Keys: s.Keys, Get: func(key string) (json.RawMessage, bool, error) { + e, err := s.Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil + }} +} + +// republishTools are builder_publish_again and builder_publish_queue (design 47 G §588). The publish again is a hand act: +// it is recorded in the hand-act log first, then written to reasks for the builder, which owns the queue, to answer +// with the versions it queued. +func republishTools(queue, acted versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool { + str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + return []stdio.Tool{ + {Name: versions.PublishAgainTool, Description: versions.PublishAgainDescription, + Input: map[string]any{"type": "object", "required": []string{"why"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "only this module's reserved versions"}, + "version": map[string]any{"type": "string", "description": "with module: only this reserved version, MAJOR.MINOR.PATCH"}, + "why": map[string]any{"type": "string", "description": "why, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + key, value, err := build.NewRepublish(str(args, "module"), str(args, "version"), str(args, "why"), stdio.CallerOf(args), now()) + if err != nil { + return nil, err + } + if _, err := acts.Record(meshhandactsclient.Request{What: fmt.Sprintf("published reserved versions again by hand (module %q, version %q)", str(args, "module"), str(args, "version")), + Why: str(args, "why"), Cause: versions.PublishAgainTool, Verb: "publish-again"}); err != nil { + return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was queued: %w", err) + } + raw, err := ask(key, value) + if err != nil { + return nil, err + } + a, err := build.DecodeReAsk(raw) + if err != nil { + return nil, err + } + if a.Refused { + return nil, errors.New(a.Answer) + } + return map[string]any{"key": key, "queued": a.Queued, "answer": a.Answer}, nil + }}, + {Name: versions.PublishQueueTool, Description: versions.PublishQueueDescription, + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { return versions.AnswerQueue(queue, acted) }}, + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index 19619406..debe2e67 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -582,18 +582,10 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 if got := reg.numberedOf("netcheck/tools-go"); len(got) != 0 { t.Errorf("version.numbered of netcheck/tools-go said %v, want none", got) } - // DLV-B10: builder_publish_again of a published, held build is refused: nothing says a version numbered (§349). for _, a := range []string{"tools-go", "tools-typescript"} { reg.published(t, "netcheck/"+a, "0.1.5", digestOf(a)) } reg.awaitOf(t, "netcheck/tools-go", func(es []versions.Entry) bool { return len(es) == 1 && es[0].State == versions.Published }) - key, again, err := build.NewPublishAgain("netcheck/tools-go", fp, "the holder lost it", "me@shanks", time.Now()) - if err != nil { - t.Fatal(err) - } - if a := reg.answered(t, key, again); !a.Refused { - t.Errorf("builder_publish_again = %+v, want a refusal", a) - } // A numbered version a holder will refuse below, written to the state as the builder numbers one. refusable, _ := json.Marshal(versions.Entry{Name: "netcheck/tools-go", Kind: versions.KindBundle, Series: "0.1", Version: "0.1.6", Module: "netcheck", Fingerprint: "src2:" + strings.Repeat("6", 64), Seat: ociartifactregistry.Seat, State: versions.Numbered, At: time.Now().UTC()}) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 407f642c..4122bc39 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -219,17 +219,22 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // A state bound later numbers the builds looked at without it. versionsKV.Ready(func(jetstream.KeyValue) { b.Reconsider(ctx) }) // The versions' hand acts (builder_publish_again, builder_floor_again) come through reasks, as builder_build_again. - acts := versions.Acts{Numberer: numberer, DropReserved: reserved.dropped, Done: doneVia(b), Reconsider: b.Reconsider, - BuildAgain: func(ctx context.Context, module, fp, why, by string) (string, bool) { - r := b.BuildAgain(ctx, module, fp, why, by, buildAgainBound) - if r.Refused != "" { - return r.Refused, true - } - if r.Failed != "" { - return "built again, and it failed before its asks were put: " + r.Failed, false - } - return fmt.Sprintf("built again as revision %d", r.Revision), false - }} + acts := versions.Acts{Numberer: numberer, DropReserved: reserved.dropped, Reconsider: b.Reconsider} + // A reserved version left unpublished is published again, by the builder on an event or by a person's tool (design 47 G + // §588): its states publish-queue and acted, bound once the bus holds them. + republishMax, err := republishMaxOf(getenv("MESH_BUILDER_REPUBLISH_MAX_FILE")) + if err != nil { + return err + } + queueKV := process.AwaitBucket(ctx, js, conn, versions.PublishQueueBucket, "no version is published again on an event or by builder_publish_again", logger.Printf) + actedKV := process.AwaitBucket(ctx, js, conn, versions.ActedBucket, "no version is published again on an event or by builder_publish_again", logger.Printf) + republish := &versions.Republisher{Reserved: reserved.r, Queue: awaitedQueue{awaitedReserved{awaitedVersions{b: queueKV, bucket: versions.PublishQueueBucket}}}, + Acted: awaitedReserved{awaitedVersions{b: actedKV, bucket: versions.ActedBucket}}, Entry: entryVia(calls.Call), Send: sendVia(calls.Call), + Unreleased: reserved.unreleasedWords, Max: republishMax, Logf: logger.Printf} + acts.Republish = republish.Republish + again := &republishing{rp: republish} + queueKV.Ready(again.bound(ctx, js, logger.Printf)) + actedKV.Ready(again.bound(ctx, js, logger.Printf)) // A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base // revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision // is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it @@ -371,6 +376,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version. seats := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved} events := func(m jetstream.Msg) { + if again.heard(ctx, m, logger.Printf) { // the publish again's own subjects, left for the bus to give again when its state cannot take them + return + } if reads.heard(m, logger.Printf) || heardTools.Reported(ctx, m) || heardTools.Running(ctx, m) || floor.heard(ctx, m) || reserved.heard(ctx, m) || seats.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4) return } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go index 26da5ebb..0acc5eb0 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go @@ -58,7 +58,7 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, kv jetstream released(ctx, b, kv, e, r, logf) return } - if r.Mode == build.ModePublishAgain || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop { + if r.Mode == build.ModeRepublish || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop { // A version's hand act (design 51 *Tools of the builder*) may build again, so it is answered beside the watch. go versionActed(ctx, acts, kv, e, r, logf) return @@ -164,10 +164,15 @@ func appendLog(ctx context.Context, js jetstream.JetStream, cred process.Account // versionActed answers a reasks value of builder_publish_again, builder_floor_again or builder_old_form's drop, a hand act said in the log. func versionActed(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) { var err error - if r.Mode == build.ModePublishAgain { - var ag versions.Again - ag, r.Answer, err = acts.PublishAgain(ctx, r.Name, r.FP, r.Why, r.By, r.At) - r.Do, r.Version = ag.Do, ag.Entry.Version + if r.Mode == build.ModeRepublish { + if acts.Republish == nil { + err = errors.New("this builder publishes nothing again") + } else { + r.Queued, err = acts.Republish(ctx, r.Module, r.Version) + if err == nil { + r.Answer = "queued " + fmt.Sprint(len(r.Queued)) + " versions to publish again: " + strings.Join(r.Queued, ", ") + "; builder_publish_queue shows them" + } + } } else if r.Mode == build.ModeReservedDrop { if acts.DropReserved == nil { err = errors.New("this builder keeps no reservations") diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go new file mode 100644 index 00000000..7d201fb3 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go @@ -0,0 +1,146 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "strconv" + "strings" + "sync/atomic" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + + "mesh-delivery-builder/internal/versions" +) + +// The builder's publish again (design 47 G §588, A-73, A-76): the wiring of versions.Republisher to the bus. + +// awaitedQueue is the state publish-queue as versions.QueueStore: bound once the bus holds it; its bytes are the cap's. +type awaitedQueue struct{ awaitedReserved } + +func (s awaitedQueue) Bytes(ctx context.Context) (uint64, error) { + kv, err := s.kv() + if err != nil { + return 0, err + } + st, err := kv.Status(ctx) + if err != nil { + return 0, err + } + return st.Bytes(), nil +} + +// entryVia reads a catalog graph entry by its fingerprint through the seat mesh-catalog's verb entry. +func entryVia(call toolCaller) func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) { + return func(_ context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) { + raw, err := call("seat:"+meshcatalog.Seat+"."+meshcatalog.VerbEntry, meshcatalog.EntryInput{Fingerprint: fingerprint}) + if err != nil { + return meshcatalog.CatalogEntry{}, err + } + var a meshcatalog.EntryAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return meshcatalog.CatalogEntry{}, fmt.Errorf("the catalog answered what is no entry: %w", err) + } + return a.Entry, nil + } +} + +// sendVia sends a publish to the registry seat with the digest and no parts (A-73): the holder takes the bytes from the +// artifact store, addressed by digest. The four seats' publish take one wire form; an answer that lacks parts, or a fault of +// the call, is the holder's refusal, said in its words (not stored when the holder lacks the bytes). +func sendVia(call toolCaller) func(ctx context.Context, p versions.Publish) error { + return func(_ context.Context, p versions.Publish) error { + raw, err := call("seat:"+p.Seat+"."+versions.VerbPublish, publishBody(p)) + if err != nil { + return fmt.Errorf("%s's publish of %s %s: %w", p.Seat, p.Name, p.Version, err) + } + var a gopackageregistry.PublishAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return fmt.Errorf("%s answered what is no publish answer: %w", p.Seat, err) + } + if len(a.Lacks) > 0 { + return fmt.Errorf("%s%s holds %d of the parts of %s %s", versions.NotStoredWords, p.Seat, len(a.Lacks), p.Name, p.Version) + } + return nil + } +} + +// publishBody is the seat's publish input for p: the digest, its kind and name and version, no parts. +func publishBody(p versions.Publish) gopackageregistry.PublishInput { + return gopackageregistry.PublishInput{Commit: p.Commit, Digest: p.Digest, Fingerprint: p.Fingerprint, Kind: p.Kind, Name: p.Name, Transfer: p.Transfer, Version: p.Version} +} + +// republishMaxOf reads the setting republish-max-versions from the file the manifest declares (design 47 G §633, A-71's +// form): no file or an empty one is the default; a content that is no positive number is an error naming the file. +func republishMaxOf(path string) (int, error) { + if path == "" { + return versions.DefaultRepublishMax, nil + } + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return versions.DefaultRepublishMax, nil + } + if err != nil { + return 0, err + } + v := strings.TrimSpace(string(raw)) + if v == "" { + return versions.DefaultRepublishMax, nil + } + n, err := strconv.Atoi(v) + if err != nil || n < 1 { + return 0, fmt.Errorf("%s holds %q, which is not a number of versions", path, v) + } + return n, nil +} + +// republishing starts the publish again once both its states are bound: the queue is taken up again and the last running +// of every node's module is read (§556), acted on where its sequence is newer than the kept one. +type republishing struct { + rp *versions.Republisher + ready atomic.Int32 +} + +func (g *republishing) bound(ctx context.Context, js jetstream.JetStream, logf func(string, ...any)) func(jetstream.KeyValue) { + return func(jetstream.KeyValue) { + if g.ready.Add(1) != 2 { + return + } + if err := g.rp.Resume(ctx); err != nil { + logf("the queued publishes are not taken up again: %v", err) + } + err := lastPerSubject(ctx, js, nodeengine.StreamRunning, nodeengine.SubjectRunningAll, func(subject string, data []byte, seq uint64) { + if _, err := g.rp.Heard(ctx, subject, data, seq); err != nil { + logf("the running %s is not acted on at the start: %v", subject, err) + } + }) + if err != nil { + logf("the running of the node-engine seat is not read for the publish again at the start: %v", err) + } + } +} + +// heard hands a message of the events consumer to the publish again: one of its subjects whose state cannot take it is left +// for the bus to give again (Nak); every message goes on to the other parts. +func (g *republishing) heard(ctx context.Context, m jetstream.Msg, logf func(string, ...any)) (stop bool) { + meta, err := m.Metadata() + if err != nil { + return false + } + handled, err := g.rp.Heard(ctx, m.Subject(), m.Data(), meta.Sequence.Stream) + if handled && err != nil { + logf("%v; heard again", err) + if nakErr := m.Nak(); nakErr != nil { + logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr) + } + return true + } + return false +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go new file mode 100644 index 00000000..0532ed69 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go @@ -0,0 +1,79 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/npmpackageregistry" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry" + + "mesh-delivery-builder/internal/build" + "mesh-delivery-builder/internal/versions" +) + +// TestRepublishSend: design 47 G §588, A-73: a publish again is the registry seat's publish with the digest and no parts; the +// four seats take one wire form, a call's fault or an answer that lacks parts is the holder's refusal in its words, and its +// setting is read from the file the manifest declares. +func TestRepublishSend(t *testing.T) { + p := versions.Publish{Seat: versions.SeatArtifacts, Kind: "bundle", Name: "m/prog", Version: "0.1.3", Digest: "sha256:aa", Fingerprint: "src2:m", Commit: "c", Transfer: "t"} + want, _ := json.Marshal(publishBody(p)) + if strings.Contains(string(want), "parts") { + t.Errorf("the publish names parts: %s", want) + } + for name, v := range map[string]any{"container image": containerimageregistry.PublishInput{Digest: "sha256:aa"}, "oci": ociartifactregistry.PublishInput{Digest: "sha256:aa"}, + "go": gopackageregistry.PublishInput{Digest: "sha256:aa"}, "npm": npmpackageregistry.PublishInput{Digest: "sha256:aa"}} { + var ref, got map[string]any + a, _ := json.Marshal(gopackageregistry.PublishInput{Digest: "sha256:aa"}) + b, _ := json.Marshal(v) + _ = json.Unmarshal(a, &ref) + _ = json.Unmarshal(b, &got) + if len(ref) != len(got) { + t.Errorf("the %s seat's publish input is %s, the go seat's %s: one wire form is sent to all four", name, b, a) + } + } + var asked string + for name, c := range map[string]struct { + answer string + err error + want string + }{ + "held": {`{"held":true,"transfer":"t"}`, nil, ""}, + "lacks": {`{"lacks":["sha256:bb"],"transfer":"t"}`, nil, versions.NotStoredWords}, + "a fault": {"", errors.New("no responders"), "no responders"}, + "no sense": {`[`, nil, "no publish answer"}, + } { + send := sendVia(func(key string, body any) (json.RawMessage, error) { + asked = key + return json.RawMessage(c.answer), c.err + }) + err := send(context.Background(), p) + if asked != "seat:oci-artifact-registry.publish" || (c.want == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.want)) { + t.Errorf("%s: asked %q, error %v, want %q", name, asked, err, c.want) + } + } + dir := t.TempDir() + for content, want := range map[string]int{"": 50, "7": 7} { + path := filepath.Join(dir, "max") + _ = os.WriteFile(path, []byte(content), 0o600) + if got, err := republishMaxOf(path); err != nil || got != want { + t.Errorf("republishMaxOf(%q) = %d, %v, want %d", content, got, err, want) + } + } + _ = os.WriteFile(filepath.Join(dir, "bad"), []byte("0"), 0o600) + if _, err := republishMaxOf(filepath.Join(dir, "bad")); err == nil { + t.Error("a setting of 0 is accepted") + } + if _, _, err := build.NewRepublish("", "0.1.3", "why", "me", versionsNow()); err == nil { + t.Error("a version with no module is accepted") + } +} + +func versionsNow() time.Time { return time.Unix(1, 0) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go index 7d286318..7dd5921d 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go @@ -125,6 +125,11 @@ func (rv *reservations) unreleased(ctx context.Context, v versions.ReservedVersi needs: "builder_reserved shows the reservation; builder_publish_again publishes the missing artifacts under it; builder_reserved_drop drops it."}) } +// unreleasedWords raises version-unreleased about a version with the words of a publish again that ended badly. +func (rv *reservations) unreleasedWords(ctx context.Context, v versions.ReservedVersion, why string) { + rv.unreleased(ctx, v, why) +} + // ended is told each change of the state bounds: a reservation's bound whose time ended raises version-unreleased while // the reservation stands built. func (rv *reservations) ended(ctx context.Context, e jetstream.KeyValueEntry) bool { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go index 62c88a80..bdcf23dd 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go @@ -254,8 +254,8 @@ func (h *toolchainsHeard) agree(ctx context.Context) { } // lastPerSubject reads the last message of every subject of a stream kept last per subject, through an ordered consumer -// made for that read (design 47 G §556), and hands each to each; it answers when the last of them was handed. -func lastPerSubject(ctx context.Context, js jetstream.JetStream, stream, filter string, each func(subject string, data []byte)) error { +// made for that read (design 47 G §556), and hands each to each with its stream sequence; it answers when the last of them was handed. +func lastPerSubject(ctx context.Context, js jetstream.JetStream, stream, filter string, each func(subject string, data []byte, sequence uint64)) error { cons, err := js.OrderedConsumer(ctx, stream, jetstream.OrderedConsumerConfig{FilterSubjects: []string{filter}, DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy}) if err != nil { return fmt.Errorf("reading %s: %w", stream, err) @@ -278,14 +278,18 @@ func lastPerSubject(ctx context.Context, js jetstream.JetStream, stream, filter if err != nil { return fmt.Errorf("reading %s: %w", stream, err) } - each(m.Subject(), m.Data()) + seq := uint64(0) + if meta, err := m.Metadata(); err == nil { + seq = meta.Sequence.Stream + } + each(m.Subject(), m.Data(), seq) } return nil } // readRunning is the start's read of every node's module's last running (§556), before the consumer's. func (h *toolchainsHeard) readRunning(ctx context.Context, js jetstream.JetStream) { - if err := lastPerSubject(ctx, js, nodeengine.StreamRunning, nodeengine.SubjectRunningAll, func(_ string, data []byte) { h.running(ctx, data) }); err != nil { + if err := lastPerSubject(ctx, js, nodeengine.StreamRunning, nodeengine.SubjectRunningAll, func(_ string, data []byte, _ uint64) { h.running(ctx, data) }); err != nil { h.logf("the running of the node-engine seat is not read at the start, so a build module stopped while this builder was away is forgotten at its next running: %v", err) } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go index ed1460ab..ff2cf8c6 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go @@ -13,7 +13,6 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-builder/internal/build" @@ -306,22 +305,3 @@ func (f *floorWake) wake(ctx context.Context, kind, seat, module string) { func conditionModules(id string) []string { return strings.FieldsFunc(id, func(r rune) bool { return r == '/' || r == '.' || r == ':' }) } - -// doneVia says whether a build key is done, read from the builder's own builds state: the build's file is held. -func doneVia(b *build.Builder) func(ctx context.Context, module, fp string) (bool, error) { - return func(ctx context.Context, module, fp string) (bool, error) { - key, err := builds.BuilderBuildKey(module, fp) - if err != nil { - return false, err - } - raw, _, found, err := b.Claimer.Store.Get(ctx, key) - if err != nil || !found { - return false, err - } - cur, err := builds.DecodeBuilderBuild(raw) - if err != nil { - return false, err - } - return cur.State == builds.BuildDone, nil - } -} diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index 26dd0fe5..3773ae2e 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -177,7 +177,7 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu for _, t := range plan.Tasks { refs = append(refs, module+":"+t.Artifact) } - src := versions.Source{Repository: m.Repository, Commit: m.Commit} + src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2} for _, t := range plan.Tasks { src.GoPackage = src.GoPackage || (t.Kind == catalogue.ArtifactPackage && t.Language == "go") } diff --git a/modules/mesh-delivery-builder/internal/build/stuck.go b/modules/mesh-delivery-builder/internal/build/stuck.go index 560bce85..eafe7379 100644 --- a/modules/mesh-delivery-builder/internal/build/stuck.go +++ b/modules/mesh-delivery-builder/internal/build/stuck.go @@ -57,35 +57,32 @@ type ReAsk struct { Failed string `json:"failed,omitempty"` Revision uint64 `json:"revision,omitempty"` Asks map[string]string `json:"asks,omitempty"` - // Name is the numbered name (`/`) of a ModePublishAgain or ModeFloorAgain value; Do and Version the - // builder's answer: what it did (versions.Again*) and the version published. - Name string `json:"name,omitempty"` - Do string `json:"do,omitempty"` - Version string `json:"version,omitempty"` + // Name is the numbered name (`/`) of a ModeFloorAgain value; Version is a ModeRepublish value's + // request and Queued the builder's answer to it: the versions it queued, ` `. + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + Queued []string `json:"queued,omitempty"` } -// ModePublishAgain is the Mode of a reasks value of builder_publish_again (design 51 *Tools of the builder*). -const ModePublishAgain = "publish-again" +// ModeRepublish is the Mode of a reasks value of builder_publish_again (design 47 G §588): Module and Version, each +// optional, name what to publish again; the builder answers with the versions it queued (Queued). +const ModeRepublish = "republish" // ModeFloorAgain is the Mode of a reasks value of builder_floor_again (design 51 *The floor*). const ModeFloorAgain = "floor-again" -// NewPublishAgain is the value builder_publish_again writes for a name's fingerprint: the key of the name's module and -// fingerprint, and the value. -func NewPublishAgain(name, fp, why, by string, at time.Time) (string, []byte, error) { - module, artifact, ok := strings.Cut(strings.TrimSpace(name), "/") - if !ok || module == "" || artifact == "" { - return "", nil, fmt.Errorf("name %q is not a numbered name, /, as builder_versions lists it", name) - } - key, err := builds.BuilderBuildKey(module, fp) - if err != nil { - return "", nil, err +// NewRepublish is the value builder_publish_again writes (design 47 G §588): its key, `republish.` +// (a call is one value, never a build key), and the value. Module and version are each optional. +func NewRepublish(module, version, why, by string, at time.Time) (string, []byte, error) { + module, version = strings.TrimSpace(module), strings.TrimSpace(version) + if version != "" && module == "" { + return "", nil, errors.New("a version is published again under its module: give module with version") } if strings.TrimSpace(why) == "" { return "", nil, errors.New("say why it is published again (why)") } - data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModePublishAgain, Module: module, FP: fp, Name: strings.TrimSpace(name)}) - return key, data, err + data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeRepublish, Module: module, Version: version}) + return "republish." + strconv.FormatInt(at.UnixNano(), 10), data, err } // NewFloorAgain is the value builder_floor_again writes for a name: the key `floor.`, never a build key, diff --git a/modules/mesh-delivery-builder/internal/versions/again.go b/modules/mesh-delivery-builder/internal/versions/again.go index 52a3decf..709007be 100644 --- a/modules/mesh-delivery-builder/internal/versions/again.go +++ b/modules/mesh-delivery-builder/internal/versions/again.go @@ -2,63 +2,13 @@ package versions import ( "context" - "errors" "fmt" "strings" "time" ) -// What builder_publish_again does with one fingerprint of a name (design 51 *Tools of the builder*), after a holder failed. -const ( - // AgainPublish is a held build whose recorded version is published again: refused since design 47 G §349 (no version.numbered is said). - AgainPublish = "publish" - // AgainBuild builds the fingerprint again under the same number: the build's file is gone and no published was said - // for the number, so the new digest replaces the recorded one (Numberer.Built). - AgainBuild = "build" - // AgainRenumber takes a new number: a published was said for the number, so other bytes never go under it. - AgainRenumber = "renumber" -) - -// Again is what builder_publish_again does, with the entry it acts on and why. -type Again struct { - Do string `json:"do"` - Entry Entry `json:"entry"` - Why string `json:"why"` -} - -// PublishAgain decides builder_publish_again for fingerprint among a name's entries; built says whether the builder still -// holds the build's file (its done key). It refuses a fingerprint not numbered, and a version refused for other bytes -// than the ones recorded under it. -func PublishAgain(entries []Entry, fingerprint string, built bool) (Again, error) { - fingerprint = strings.TrimSpace(fingerprint) - if fingerprint == "" { - return Again{}, errors.New("fp is required: the fingerprint whose version is published again") - } - for _, e := range latestFirst(entries) { - if e.Tag != "" || e.Fingerprint != fingerprint { - continue - } - switch { - case e.State == Refused && e.RefusedDigest != "" && e.Digest != "" && e.RefusedDigest != e.Digest: - return Again{}, fmt.Errorf("%s@%s was refused for other bytes (%s, recorded %s): it is not published again; a new build takes a new number", - e.Name, e.Version, e.RefusedDigest, e.Digest) - case e.State == Published || e.State == Removed: - return Again{Do: AgainRenumber, Entry: e, Why: fmt.Sprintf("%s@%s was %s, so the fingerprint takes a new number", e.Name, e.Version, e.State)}, nil - case !built: - return Again{Do: AgainBuild, Entry: e, Why: fmt.Sprintf("the build's file is gone and no published was said for %s@%s: built again under the same number", e.Name, e.Version)}, nil - } - return Again{Do: AgainPublish, Entry: e, Why: fmt.Sprintf("%s@%s (%s) is held", e.Name, e.Version, e.State)}, nil - } - return Again{}, fmt.Errorf("fingerprint %s is not numbered for this name: builder_versions lists what is", fingerprint) -} - // The tools' descriptions. const ( - PublishAgainDescription = "Publish one fingerprint's version again after its holder failed (design 51): with the build's file held, the " + - "nothing says a version numbered since design 47 G §349, so a held build is refused; with the file gone, it builds the fingerprint " + - "again under the same number, the new digest replacing the recorded one, since no published was said for it; after a published (or " + - "a removal) it takes a new number. It refuses a fingerprint not numbered and a version the holder refused for other bytes. A hand " + - "act, recorded with its caller and why in the builder's log." FloorAgainDescription = "Ask the floor of one name again now, by hand (design 51 *The floor*): the seat's versions, the forge bridge's " + "bridge_tags and the indexer's indexer_requirements, as the builder asks them by itself when a silent source says something. " + "Answers which sources were silent and whether the floor is now known. A name the builder's versions already hold needs no floor: " + @@ -68,56 +18,16 @@ const ( // Acts are the builder's hand acts on its versions, answered by its program (the tools bundle writes them to reasks). type Acts struct { Numberer *Numberer - // Done says whether the build key of module and fingerprint is done: the build's file is held. - Done func(ctx context.Context, module, fp string) (bool, error) - // BuildAgain builds a failed key again (builder_build_again's path): its answer, or why it refused. - BuildAgain func(ctx context.Context, module, fp, why, by string) (answer string, refused bool) // Reconsider looks at every module again, so a claim asks a name's floor. Reconsider func(ctx context.Context) + // Republish queues the reserved versions builder_publish_again names (design 47 G §588): the versions it queued. + Republish func(ctx context.Context, module, version string) ([]string, error) // DropOldForm drops a retired node from the old form's count by hand (builder_old_form): its answer, or why not. DropOldForm func(ctx context.Context, node, by, why string, at time.Time) (string, error) // DropReserved drops a module's reservation of one MAJOR.MINOR by hand (builder_reserved_drop): its answer, or why not. DropReserved func(ctx context.Context, module, series, by, why string, at time.Time) (string, error) } -// PublishAgain is builder_publish_again of name's fingerprint: what it did, the version, and its answer; an error refuses. -func (a Acts) PublishAgain(ctx context.Context, name, fp, why, by string, at time.Time) (Again, string, error) { - module, _, _ := strings.Cut(name, "/") - entries, err := a.Numberer.Versions.Of(ctx, name) - if err != nil { - return Again{}, "", err - } - done, err := a.Done(ctx, module, fp) - if err != nil { - return Again{}, "", fmt.Errorf("the build key of %s at %s cannot be read: %w", module, fp, err) - } - ag, err := PublishAgain(entries, fp, done) - if err != nil { - return Again{}, "", err - } - if ag.Do == AgainPublish || (ag.Do == AgainRenumber && done) { - return Again{}, "", errors.New("nothing says a version numbered since design 47 G §349: the version is published under its reservation (§369)") - } - e := ag.Entry - if ag.Do == AgainRenumber { - _, ne, err := a.Numberer.Versions.Number(ctx, Claim{Name: e.Name, Kind: e.Kind, Series: e.Series, Module: e.Module, Fingerprint: e.Fingerprint, - Commit: e.Commit, Revision: e.Revision, Seat: e.Seat, Merge: e.Merge, Fresh: true}) - if err != nil { - return Again{}, "", err - } - ag.Entry = ne - } - answer := ag.Why - if ag.Do == AgainBuild || (ag.Do == AgainRenumber && !done) { - said, refused := a.BuildAgain(ctx, module, fp, why, by) - if refused { - return Again{}, "", fmt.Errorf("%s; and the build again was refused: %s", ag.Why, said) - } - answer += "; " + said - } - return ag, answer, nil -} - // FloorAgain is builder_floor_again of name: the floor counts only for a name the versions hold nothing of. func (a Acts) FloorAgain(ctx context.Context, name string) (string, error) { entries, err := a.Numberer.Versions.Of(ctx, name) diff --git a/modules/mesh-delivery-builder/internal/versions/republish.go b/modules/mesh-delivery-builder/internal/versions/republish.go new file mode 100644 index 00000000..07dc30cc --- /dev/null +++ b/modules/mesh-delivery-builder/internal/versions/republish.go @@ -0,0 +1,646 @@ +package versions + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// **A reserved version left unpublished is published again by the builder, on an event or by a person's tool** (design 47 +// G §588, ADR 0674): on the node-engine seat's `running` of a module whose catalog graph entry claims the registry seat +// of the missing artifacts' kind, and on its `manifest-reads-changed` with `why` `returned` of the node such a holder +// runs on. Each message is acted on once, by its stream sequence (state `acted`); each holder has one queue of versions, +// kept in the state `publish-queue`, sent one at a time. A send is the registry seat's `publish` with the digest and no +// parts (A-73): the holder takes the bytes from the artifact store by digest, and refuses what no store holds. + +const ( + // PublishQueueState is the builder's old state section of the queues, capped at PublishQueueMiB. + PublishQueueState = "publish-queue" + PublishQueueBucket = "mesh-delivery-builder_" + PublishQueueState + PublishQueueMiB = 4 + // ActedState holds, one key per subject, the last stream sequence the builder acted on; capped at ActedMiB. + ActedState = "acted" + ActedBucket = "mesh-delivery-builder_" + ActedState + ActedMiB = 1 + // PublishQueueTool is builder_publish_queue. + PublishQueueTool = "builder_publish_queue" + // DefaultRepublishMax is the setting republish-max-versions unless set (§633). + DefaultRepublishMax = 50 + // QueueFull is what version-unreleased says of a version the queue's cap refused. + QueueFull = "queue full" + // NotStoredWords opens what a refused artifact is said as. + NotStoredWords = "not stored: " + // VerbPublish is the registry seats' verb a publish again sends. + VerbPublish = "publish" +) + +// RegistrySeats are the four registry seats a version is published to. +var RegistrySeats = []string{SeatImages, SeatArtifacts, SeatGoPackages, SeatNPM} + +// SeatOfArtifact is the registry seat of one artifact of the catalog graph entry: from its package's ecosystem when the +// entry names one (go, npm, image), else from its kind (SeatOf). +func SeatOfArtifact(a meshcatalog.Artifact) (string, bool) { + if a.Package != nil { + switch a.Package.Ecosystem { + case "go": + return SeatGoPackages, true + case "npm": + return SeatNPM, true + case "image": + return SeatImages, true + } + } + seat, _, ok := SeatOf(a.Kind, a.Language) + return seat, ok +} + +// QueuedPublish is the value of a publish-queue key `..` (each token escaped, §73): the +// artifacts of the version this holder publishes, by reference with their digests, when it was queued and whether its +// publish has begun. +type QueuedPublish struct { + V int `json:"v"` + Holder string `json:"holder"` + Module string `json:"module"` + Version string `json:"version"` + Artifacts map[string]string `json:"artifacts"` + Queued time.Time `json:"queued"` + Begun bool `json:"begun,omitempty"` +} + +// QueueKey is the key of holder's queued version of module. +func QueueKey(holder, module, version string) string { + return EscapeToken(holder) + "." + EscapeToken(module) + "." + EscapeToken(version) +} + +// Acted is the value of an acted key: the last stream sequence acted on for its subject. +type Acted struct { + Sequence uint64 `json:"sequence"` +} + +// QueueStore is the queue's bucket: compare-and-set, deletion and how many bytes it holds, for its cap. +type QueueStore interface { + ReserveStore + Bytes(ctx context.Context) (uint64, error) +} + +// Publish is one artifact's publish again: what the registry seat's publish takes, without parts. +type Publish struct { + Seat, Kind, Name, Version, Digest, Fingerprint, Commit, Transfer string +} + +// Republisher publishes reserved versions again (design 47 G §588). +type Republisher struct { + Reserved *Reserver + Queue QueueStore + Acted ReserveStore + // Entry reads the catalog graph entry of a fingerprint (the seat mesh-catalog's verb entry). + Entry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) + // Send sends one publish to a registry seat; an error is the holder's refusal or the call's fault, in its words. + Send func(ctx context.Context, p Publish) error + // Unreleased says version-unreleased about a version with the holder's words. + Unreleased func(ctx context.Context, v ReservedVersion, why string) + // Max is the setting republish-max-versions: the most versions one event queues. + Max int + Now func() time.Time + Logf func(string, ...any) + // Go starts the sending of a holder's queue; nil starts a goroutine. + Go func(func()) + + mu sync.Mutex + runs map[string]map[string][]string // node -> holder module -> the registry seats it claims + busy map[string]bool +} + +func (r *Republisher) now() time.Time { + if r.Now != nil { + return r.Now().UTC() + } + return time.Now().UTC() +} + +func (r *Republisher) logf(f string, a ...any) { + if r.Logf != nil { + r.Logf(f, a...) + } +} + +// ActedKey is the acted key of a subject. +func ActedKey(subject string) string { return EscapeToken(subject) } + +// Heard takes one message of the node-engine seat's running or manifest-reads-changed in, at stream sequence seq: it acts +// on it only when its sequence is newer than the one kept for its subject, and keeps it after. A subject that is neither is +// not its; an error leaves the message not acted on, so it is read again. +func (r *Republisher) Heard(ctx context.Context, subject string, data []byte, seq uint64) (bool, error) { + running := strings.HasPrefix(subject, strings.TrimSuffix(nodeengine.SubjectRunningAll, "*.*")) + changed := strings.HasPrefix(subject, "mesh.seat.node-engine.event."+nodeengine.EventManifestReadsChanged+".") + if !running && !changed { + return false, nil + } + key := ActedKey(subject) + raw, rev, found, err := r.Acted.Get(ctx, key) + if err != nil { + return true, fmt.Errorf("the acted sequence of %s cannot be read: %w", subject, err) + } + if found { + var a Acted + if json.Unmarshal(raw, &a) == nil && a.Sequence >= seq { + return true, nil + } + } + if running { + err = r.running(ctx, data) + } else { + err = r.returned(ctx, data) + } + if err != nil { + return true, err + } + val, _ := json.Marshal(Acted{Sequence: seq}) + if found { + _, err = r.Acted.Update(ctx, key, val, rev) + } else { + _, err = r.Acted.Create(ctx, key, val) + } + if err != nil && !errors.Is(err, ErrConflict) { + return true, fmt.Errorf("the acted sequence of %s cannot be kept: %w", subject, err) + } + return true, nil +} + +// registrySeatsOf are the registry seats the module of the catalog entry fingerprint claims, by their publish verb. +func (r *Republisher) registrySeatsOf(ctx context.Context, fingerprint string) ([]string, error) { + e, err := r.Entry(ctx, fingerprint) + if err != nil { + return nil, err + } + var out []string + for _, seat := range RegistrySeats { + if slices.Contains(e.Serves, seat+"."+VerbPublish) { + out = append(out, seat) + } + } + return out, nil +} + +// running takes a running in: a module stopped is forgotten; one running whose entry claims a registry seat is a holder on +// its node, and its versions are queued. +func (r *Republisher) running(ctx context.Context, data []byte) error { + var ev nodeengine.Running + if err := json.Unmarshal(data, &ev); err != nil || ev.Module == "" || ev.Node == "" { + r.logf("a running that does not read is not acted on: %v", err) + return nil + } + if ev.Stopped || ev.Fingerprint == "" { + r.mu.Lock() + delete(r.runs[ev.Node], ev.Module) + r.mu.Unlock() + return nil + } + seats, err := r.registrySeatsOf(ctx, ev.Fingerprint) + if err != nil { + return fmt.Errorf("the catalog entry %s of %s cannot be read: %w", ev.Fingerprint, ev.Module, err) + } + r.mu.Lock() + if r.runs == nil { + r.runs = map[string]map[string][]string{} + } + if r.runs[ev.Node] == nil { + r.runs[ev.Node] = map[string][]string{} + } + if len(seats) == 0 { + delete(r.runs[ev.Node], ev.Module) + } else { + r.runs[ev.Node][ev.Module] = seats + } + r.mu.Unlock() + if len(seats) == 0 { + return nil + } + _, err = r.queueEvent(ctx, map[string][]string{ev.Module: seats}) + return err +} + +// returned takes a manifest-reads-changed in: only one with why returned acts, for every holder known to run on its node. +func (r *Republisher) returned(ctx context.Context, data []byte) error { + var ev nodeengine.ManifestReadsChanged + if err := json.Unmarshal(data, &ev); err != nil || ev.Node == "" { + r.logf("a manifest-reads-changed that does not read is not acted on: %v", err) + return nil + } + if ev.Why != nodeengine.ManifestReadsChangedWhyReturned { + return nil + } + r.mu.Lock() + holders := map[string][]string{} + for m, seats := range r.runs[ev.Node] { + holders[m] = seats + } + r.mu.Unlock() + if len(holders) == 0 { + return nil + } + _, err := r.queueEvent(ctx, holders) + return err +} + +// view is a reserved version with the artifacts of it a holder would publish. +type view struct { + v ReservedVersion + arts map[string]meshcatalog.Artifact // reference -> its catalog artifact +} + +// queueEvent queues, for each holder (module -> the seats it claims), the unreleased versions whose missing artifacts +// that holder's seats take: newest keys first, at most Max versions in all (one key per module and MAJOR.MINOR is the +// reservation's own). +func (r *Republisher) queueEvent(ctx context.Context, holders map[string][]string) ([]string, error) { + over, err := r.Reserved.Overdue(ctx, nil) + if err != nil { + return nil, err + } + var vs []ReservedVersion + for _, o := range over { + if v, ok, err := r.Reserved.Of(ctx, o.Module, o.Version); err == nil && ok { + vs = append(vs, v) + } + } + return r.queue(ctx, vs, func(seat string) []string { return holdersOf(holders, seat) }, r.Max) +} + +func holdersOf(holders map[string][]string, seat string) []string { + var out []string + for m, seats := range holders { + if slices.Contains(seats, seat) { + out = append(out, m) + } + } + sort.Strings(out) + return out +} + +// queue queues vs newest first, at most max of them (0 is no limit), each artifact to the first holder holdersOf names for +// its seat. It answers the versions queued, ` `. +func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf func(seat string) []string, max int) ([]string, error) { + sort.SliceStable(vs, func(i, j int) bool { return vs[i].At.After(vs[j].At) }) + var queued []string + started := map[string]bool{} + for _, v := range vs { + if max > 0 && len(queued) >= max { + break + } + if v.Fingerprint == "" || v.BuiltAt.IsZero() { + continue + } + e, err := r.Entry(ctx, v.Fingerprint) + if err != nil { + return queued, fmt.Errorf("the catalog entry %s of %s %s cannot be read: %w", v.Fingerprint, v.Module, v.Version, err) + } + byHolder := map[string]map[string]string{} + for _, ref := range v.Missing() { + if _, no := v.Unstored[ref]; no { + continue + } + for _, a := range e.Artifacts { + seat, ok := SeatOfArtifact(a) + if a.Ref != ref || !ok { + continue + } + if hs := holdersOf(seat); len(hs) > 0 { + if byHolder[hs[0]] == nil { + byHolder[hs[0]] = map[string]string{} + } + byHolder[hs[0]][ref] = v.Artifacts[ref] + } + } + } + did := false + for holder, arts := range byHolder { + ok, err := r.enqueue(ctx, v, holder, arts) + if err != nil { + return queued, err + } + did = did || ok + started[holder] = true + } + if did { + queued = append(queued, v.Module+" "+v.Version) + } + } + for holder := range started { + r.drainLater(ctx, holder) + } + return queued, nil +} + +// enqueue puts one holder's artifacts of a version in its queue: not again when it is queued already, and not past the +// section's cap (said as queue full). +func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder string, arts map[string]string) (bool, error) { + key := QueueKey(holder, v.Module, v.Version) + if _, _, found, err := r.Queue.Get(ctx, key); err != nil { + return false, err + } else if found { + return false, nil + } + raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.now()}) + used, err := r.Queue.Bytes(ctx) + if err != nil { + return false, fmt.Errorf("the size of %s cannot be read: %w", PublishQueueBucket, err) + } + if used+uint64(len(raw)) > PublishQueueMiB<<20 { + if r.Unreleased != nil { + r.Unreleased(ctx, v, QueueFull) + } + return false, nil + } + if _, err := r.Queue.Create(ctx, key, raw); err != nil { + if errors.Is(err, ErrConflict) { + return false, nil + } + return false, fmt.Errorf("%s cannot be queued: %w", key, err) + } + return true, nil +} + +// Republish is builder_publish_again: with module and version, that reserved version's missing artifacts; with module +// alone, each reserved version of it whose artifacts are not all published; with neither, every version the finding +// version-unreleased names. It queues as the events do (no limit but the cap), for the holders known to run, and answers +// the versions it queued. +func (r *Republisher) Republish(ctx context.Context, module, version string) ([]string, error) { + var vs []ReservedVersion + if module == "" { + over, err := r.Reserved.Overdue(ctx, nil) + if err != nil { + return nil, err + } + for _, o := range over { + if v, ok, err := r.Reserved.Of(ctx, o.Module, o.Version); err == nil && ok { + vs = append(vs, v) + } + } + } else { + all, err := r.Reserved.All(ctx) + if err != nil { + return nil, err + } + for _, v := range all { + if v.Module == module && (version == "" || v.Version == version) && len(v.Missing()) > 0 { + vs = append(vs, v) + } + } + if len(vs) == 0 { + return nil, fmt.Errorf("%s holds no reserved version%s with an artifact unpublished", module, map[bool]string{true: " " + version, false: ""}[version != ""]) + } + } + r.mu.Lock() + known := map[string][]string{} + for _, mods := range r.runs { + for m, seats := range mods { + known[m] = seats + } + } + r.mu.Unlock() + return r.queue(ctx, vs, func(seat string) []string { return holdersOf(known, seat) }, 0) +} + +// Queued are the queued versions, per holder in the order they are sent, and the acted keys with their sequences: +// builder_publish_queue. +type Queued struct { + Queue []QueuedPublish `json:"queue"` + Acted map[string]uint64 `json:"acted"` +} + +// AnswerQueue is builder_publish_queue over the two sections as the tools bundle reads them. +func AnswerQueue(queue, acted Reader) (Queued, error) { + out := Queued{Queue: []QueuedPublish{}, Acted: map[string]uint64{}} + keys, err := queue.Keys() + if err != nil { + return out, fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err) + } + for _, k := range keys { + raw, ok, err := queue.Get(k) + if err != nil || !ok { + continue + } + var q QueuedPublish + if json.Unmarshal(raw, &q) == nil { + out.Queue = append(out.Queue, q) + } + } + sort.SliceStable(out.Queue, func(i, j int) bool { + if out.Queue[i].Holder != out.Queue[j].Holder { + return out.Queue[i].Holder < out.Queue[j].Holder + } + return out.Queue[i].Queued.Before(out.Queue[j].Queued) + }) + akeys, err := acted.Keys() + if err != nil { + return out, fmt.Errorf("the state %s cannot be listed: %w", ActedState, err) + } + for _, k := range akeys { + raw, ok, err := acted.Get(k) + if err != nil || !ok { + continue + } + var a Acted + if json.Unmarshal(raw, &a) == nil { + out.Acted[k] = a.Sequence + } + } + return out, nil +} + +// PublishQueueDescription is builder_publish_queue's description. +const PublishQueueDescription = "The versions the builder will publish again (its state " + PublishQueueState + ", design 47 G section 588): per registry " + + "seat holder, in the order they are sent, each with its artifacts' digests, when it was queued and whether its publish has begun; and the " + + "keys of the state " + ActedState + ", the last stream sequence the builder acted on for each running and manifest-reads-changed subject. Read-only." + +// PublishAgainDescription is builder_publish_again's description. +const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " + + "version's missing artifacts; with module alone, each reserved version of that module whose artifacts are not all published; with neither, every " + + "version the finding version-unreleased names, newest first. It queues them as the builder does on a registry holder's running, and answers at once " + + "with the versions it queued; each outcome is said as package-published, or left in version-unreleased with the holder's words. A hand act, " + + "recorded in the hand-act log." + +// Resume takes up, at the builder's start, every queued version again, a begun one sent anew (§588). +func (r *Republisher) Resume(ctx context.Context) error { + keys, err := r.Queue.KeysOf(ctx, AllFilter) + if err != nil { + return fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err) + } + holders := map[string]bool{} + for _, k := range keys { + raw, _, found, err := r.Queue.Get(ctx, k) + if err != nil || !found { + continue + } + var q QueuedPublish + if json.Unmarshal(raw, &q) == nil { + holders[q.Holder] = true + } + } + for h := range holders { + r.drainLater(ctx, h) + } + return nil +} + +func (r *Republisher) drainLater(ctx context.Context, holder string) { + r.mu.Lock() + if r.busy == nil { + r.busy = map[string]bool{} + } + if r.busy[holder] { + r.mu.Unlock() + return + } + r.busy[holder] = true + r.mu.Unlock() + run := func() { + defer func() { r.mu.Lock(); delete(r.busy, holder); r.mu.Unlock() }() + r.Drain(ctx, holder) + } + if r.Go != nil { + r.Go(run) + return + } + go run() +} + +// Drain sends holder's queued versions one at a time, oldest first, each bounded by UnreleasedBound; a key is deleted when +// its outcome is said. +func (r *Republisher) Drain(ctx context.Context, holder string) { + for ctx.Err() == nil { + keys, err := r.Queue.KeysOf(ctx, EscapeToken(holder)+".*.*") + if err != nil { + r.logf("the queue of %s cannot be listed: %v", holder, err) + return + } + var items []QueuedPublish + for _, k := range keys { + if raw, _, found, err := r.Queue.Get(ctx, k); err == nil && found { + var q QueuedPublish + if json.Unmarshal(raw, &q) == nil { + items = append(items, q) + } + } + } + if len(items) == 0 { + return + } + sort.SliceStable(items, func(i, j int) bool { return items[i].Queued.Before(items[j].Queued) }) + r.send(ctx, items[0]) + } +} + +// send publishes one queued version's artifacts and says the outcome, then deletes its key. +func (r *Republisher) send(ctx context.Context, q QueuedPublish) { + key := QueueKey(q.Holder, q.Module, q.Version) + defer func() { + if err := r.Queue.Delete(context.WithoutCancel(ctx), key); err != nil { + r.logf("%s cannot be deleted from %s: %v", key, PublishQueueState, err) + } + }() + if !q.Begun { + q.Begun = true + raw, _ := json.Marshal(q) + if _, rev, found, err := r.Queue.Get(ctx, key); err == nil && found { + _, _ = r.Queue.Update(ctx, key, raw, rev) + } + } + v, found, err := r.Reserved.Of(ctx, q.Module, q.Version) + if err != nil || !found { + return // released meanwhile, or dropped + } + e, err := r.Entry(ctx, v.Fingerprint) + if err != nil { + r.say(ctx, v, fmt.Sprintf("the catalog entry %s cannot be read: %v", v.Fingerprint, err)) + return + } + refs := make([]string, 0, len(q.Artifacts)) + for ref := range q.Artifacts { + refs = append(refs, ref) + } + sort.Strings(refs) + for _, ref := range refs { + var art *meshcatalog.Artifact + for i := range e.Artifacts { + if e.Artifacts[i].Ref == ref { + art = &e.Artifacts[i] + } + } + seat, ok := "", false + if art != nil { + seat, ok = SeatOfArtifact(*art) + } + if !ok { + r.say(ctx, v, ref+": the catalog entry names no registry seat for it") + return + } + name := NameOf(q.Module, strings.TrimPrefix(ref, q.Module+":")) + if art.Package != nil && art.Package.Name != "" { + name = art.Package.Name + } + kind := art.Kind + bound, cancel := context.WithTimeout(ctx, UnreleasedBound) // a context's bound, no timer (ADR 0437 §8) + err := r.Send(bound, Publish{Seat: seat, Kind: kind, Name: name, Version: v.Version, Digest: q.Artifacts[ref], Fingerprint: v.Fingerprint, + Commit: v.Commit, Transfer: "republish-" + q.Module + "-" + v.Version + "-" + strings.TrimPrefix(ref, q.Module+":")}) + cancel() + if err != nil { + words := err.Error() + if strings.Contains(words, strings.TrimSuffix(NotStoredWords, ": ")) { + if ok, merr := r.Reserved.MarkUnstored(ctx, q.Module, q.Version, ref, words); merr != nil { + r.logf("%s %s: the unstored %s is not kept: %v", q.Module, q.Version, ref, merr) + } else if ok { + r.say(ctx, v, ref+" "+NotStoredWords+words) + } + continue + } + r.say(ctx, v, ref+": "+words) + } + } +} + +func (r *Republisher) say(ctx context.Context, v ReservedVersion, why string) { + r.logf("%s %s is not published again: %s", v.Module, v.Version, why) + if r.Unreleased != nil { + r.Unreleased(ctx, v, why) + } +} + +// MarkUnstored records that a holder refused artifact of module's reserved version as not stored; it answers whether that +// was new (said once). +func (r *Reserver) MarkUnstored(ctx context.Context, module, version, artifact, words string) (bool, error) { + series, _, err := SplitVersion(version) + if err != nil { + return false, err + } + key := ReservedKey(module, series) + for { + cur, rev, found, err := r.get(ctx, key) + if err != nil || !found || cur.Version != version { + return false, err + } + if _, was := cur.Unstored[artifact]; was { + return false, nil + } + if cur.Unstored == nil { + cur.Unstored = map[string]string{} + } + cur.Unstored[artifact] = words + raw, _ := json.Marshal(cur) + if _, err := r.Store.Update(ctx, key, raw, rev); errors.Is(err, ErrConflict) { + continue + } else if err != nil { + return false, fmt.Errorf("%s cannot be written: %w", key, err) + } + return true, nil + } +} diff --git a/modules/mesh-delivery-builder/internal/versions/republish_test.go b/modules/mesh-delivery-builder/internal/versions/republish_test.go new file mode 100644 index 00000000..c7f15d7d --- /dev/null +++ b/modules/mesh-delivery-builder/internal/versions/republish_test.go @@ -0,0 +1,220 @@ +package versions + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +type queueMem struct { + *reserveMem + bytes uint64 +} + +func (q *queueMem) Bytes(context.Context) (uint64, error) { return q.bytes, nil } + +// republishBed is a Republisher over memory states: reservations of module m (a bundle by the oci seat and a Go package by +// the Go seat), one holder module `oci` that claims the oci seat, and a Send and a Unreleased that record. +type republishBed struct { + *Republisher + reserved *reserveMem + queue *queueMem + acted *reserveMem + now time.Time + sent []Publish + said []string + refuse string // the words Send fails with, empty for none +} + +func newRepublishBed(t *testing.T, max int) *republishBed { + t.Helper() + b := &republishBed{reserved: &reserveMem{mem: newMem()}, queue: &queueMem{reserveMem: &reserveMem{mem: newMem()}}, acted: &reserveMem{mem: newMem()}, now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)} + res := &Reserver{Store: b.reserved, Now: func() time.Time { return b.now }, ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }} + b.Republisher = &Republisher{Reserved: res, Queue: b.queue, Acted: b.acted, Max: max, Now: func() time.Time { return b.now }, Go: func(f func()) { f() }, + Entry: func(_ context.Context, fp string) (meshcatalog.CatalogEntry, error) { + if fp == "src2:holder" { + return meshcatalog.CatalogEntry{Serves: []string{SeatArtifacts + ".publish", SeatArtifacts + ".versions"}}, nil + } + m := strings.TrimPrefix(fp, "src2:") + return meshcatalog.CatalogEntry{Artifacts: []meshcatalog.Artifact{ + {Ref: m + ":prog", Kind: "bundle", Language: "go"}, + {Ref: m + ":lib", Kind: "package", Language: "go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/" + m + "/go"}}}}, nil + }, + Send: func(_ context.Context, p Publish) error { + b.sent = append(b.sent, p) + if b.refuse != "" { + return errors.New(b.refuse) + } + return nil + }, + Unreleased: func(_ context.Context, v ReservedVersion, why string) { + b.said = append(b.said, v.Module+" "+v.Version+": "+why) + }, + Logf: t.Logf} + return b +} + +// reserve reserves and builds a version of module m (its patch given by the reservation order), its build ended long ago. +func (b *republishBed) reserve(t *testing.T, module string) ReservedVersion { + t.Helper() + b.now = b.now.Add(time.Minute) + ctx := context.Background() + v, err := b.Reserved.Reserve(ctx, module, "0.1", []string{module + ":prog", module + ":lib"}, Source{Fingerprint: "src2:" + module}) + if err != nil { + t.Fatal(err) + } + for ref, d := range map[string]string{module + ":prog": "sha256:aa", module + ":lib": "sha256:bb"} { + if _, err := b.Reserved.Built(ctx, module, v.Version, ref, d); err != nil { + t.Fatal(err) + } + } + b.now = b.now.Add(UnreleasedBound + time.Minute) + return v +} + +func running(node, module, fp string, stopped bool) (string, []byte) { + data, _ := json.Marshal(nodeengine.Running{Node: node, Module: module, Fingerprint: fp, Stopped: stopped, At: time.Unix(1, 0)}) + return nodeengine.SubjectRunning(node, module), data +} + +// TestRepublish: design 47 G §588 (A-73, A-76): a reserved version left unpublished is published again, on a registry +// holder's running and on its node's returned, once per message by its stream sequence, as a publish by digest with no parts, +// at most the setting's number per event, newest first; an artifact no store holds is said once and not tried again. +func TestRepublish(t *testing.T) { + ctx := context.Background() + t.Run("running of a holder, once per sequence", func(t *testing.T) { + b := newRepublishBed(t, 50) + v := b.reserve(t, "m") + if v.Fingerprint != "src2:m" { + t.Fatalf("the reservation keeps the entry's fingerprint: %q", v.Fingerprint) + } + subject, data := running("n1", "oci", "src2:holder", false) + for _, seq := range []uint64{5, 4} { // 4 is older than the acted 5: it starts nothing + if _, err := b.Heard(ctx, subject, data, seq); err != nil || len(b.sent) != 1 { + t.Fatalf("seq %d: sent %d (%v), want 1", seq, len(b.sent), err) + } + } + p := b.sent[0] + if p.Seat != SeatArtifacts || p.Name != "m/prog" || p.Kind != "bundle" || p.Version != v.Version || p.Digest != "sha256:aa" || p.Fingerprint != "src2:m" { + t.Errorf("sent %+v, want the bundle by digest to the oci seat, the Go package left to its own seat's holder", p) + } + if _, err := b.Heard(ctx, subject, data, 5); err != nil || len(b.sent) != 1 { + t.Errorf("the same sequence read again sent %d, want still 1", len(b.sent)) + } + if _, err := b.Heard(ctx, subject, data, 6); err != nil || len(b.sent) != 2 { + t.Errorf("a newer sequence sent %d, want 2", len(b.sent)) + } + if keys, _ := b.queue.KeysOf(ctx, AllFilter); len(keys) != 0 { + t.Errorf("a key stays in the queue after its outcome: %v", keys) + } + }) + t.Run("returned acts, started does not", func(t *testing.T) { + b := newRepublishBed(t, 50) + b.reserve(t, "m") + subject, data := running("n1", "oci", "src2:holder", true) // stopped: not a holder yet + _, _ = b.Heard(ctx, subject, data, 1) + subject, data = running("n1", "oci", "src2:holder", false) + _, _ = b.Heard(ctx, subject, data, 2) + b.sent = nil + for i, why := range []string{nodeengine.ManifestReadsChangedWhyStarted, "", nodeengine.ManifestReadsChangedWhyReturned} { + ev, _ := json.Marshal(nodeengine.ManifestReadsChanged{Node: "n1", Why: why}) + handled, err := b.Heard(ctx, nodeengine.SubjectManifestReadsChanged("n1"), ev, uint64(10+i)) + if !handled || err != nil { + t.Fatalf("%q: handled %v, %v", why, handled, err) + } + if want := map[string]int{"started": 0, "": 0, "returned": 1}[why]; len(b.sent) != want { + t.Errorf("why %q: sent %d, want %d", why, len(b.sent), want) + } + } + if handled, _ := b.Heard(ctx, "mesh.seat.node-engine.event.composed.n1", nil, 1); handled { + t.Error("a subject that is neither is taken as its own") + } + }) + t.Run("the setting limits an event, newest first", func(t *testing.T) { + b := newRepublishBed(t, 2) + var versions []string + for _, m := range []string{"m1", "m2", "m3"} { + versions = append(versions, b.reserve(t, m).Module) + } + subject, data := running("n1", "oci", "src2:holder", false) + if _, err := b.Heard(ctx, subject, data, 1); err != nil { + t.Fatal(err) + } + if len(b.sent) != 2 || !strings.HasPrefix(b.sent[0].Name, "m3/") || !strings.HasPrefix(b.sent[1].Name, "m2/") { + t.Errorf("sent %+v, want the two newest, m3 then m2", b.sent) + } + }) + t.Run("an artifact no store holds is said once and not tried again", func(t *testing.T) { + b := newRepublishBed(t, 50) + b.reserve(t, "m") + b.refuse = "oci-artifact-registry's publish of m/prog: not stored: no blob sha256:aa" + subject, data := running("n1", "oci", "src2:holder", false) + _, _ = b.Heard(ctx, subject, data, 1) + _, _ = b.Heard(ctx, subject, data, 2) + if len(b.sent) != 1 || len(b.said) != 1 || !strings.Contains(b.said[0], "not stored: ") { + t.Errorf("sent %d, said %v: want one try and one saying of not stored", len(b.sent), b.said) + } + }) + t.Run("a refusal in other words is said each time", func(t *testing.T) { + b := newRepublishBed(t, 50) + b.reserve(t, "m") + b.refuse = "the holder is full" + subject, data := running("n1", "oci", "src2:holder", false) + _, _ = b.Heard(ctx, subject, data, 1) + _, _ = b.Heard(ctx, subject, data, 2) + if len(b.sent) != 2 || len(b.said) != 2 { + t.Errorf("sent %d, said %v: want both events tried and said", len(b.sent), b.said) + } + }) + t.Run("a queue past its cap says queue full", func(t *testing.T) { + b := newRepublishBed(t, 50) + b.reserve(t, "m") + b.queue.bytes = PublishQueueMiB << 20 + subject, data := running("n1", "oci", "src2:holder", false) + _, _ = b.Heard(ctx, subject, data, 1) + if len(b.sent) != 0 || len(b.said) != 1 || !strings.HasSuffix(b.said[0], QueueFull) { + t.Errorf("sent %d, said %v, want nothing sent and queue full said", len(b.sent), b.said) + } + }) + t.Run("a restart sends the queued again, a begun one anew", func(t *testing.T) { + b := newRepublishBed(t, 50) + v := b.reserve(t, "m") + raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "m", Version: v.Version, Artifacts: map[string]string{"m:prog": "sha256:aa"}, Queued: b.now, Begun: true}) + if _, err := b.queue.Create(ctx, QueueKey("oci", "m", v.Version), raw); err != nil { + t.Fatal(err) + } + if err := b.Resume(ctx); err != nil || len(b.sent) != 1 { + t.Errorf("resume sent %d (%v), want the begun version sent anew", len(b.sent), err) + } + }) + t.Run("the tool queues for a holder known to run", func(t *testing.T) { + b := newRepublishBed(t, 1) + b.reserve(t, "m1") + b.reserve(t, "m2") + if q, err := b.Republish(ctx, "", ""); err != nil || len(q) != 0 { + t.Errorf("no holder known: queued %v (%v), want none", q, err) + } + subject, data := running("n1", "oci", "src2:holder", false) + _, _ = b.Heard(ctx, subject, data, 1) // Max 1: only m2 sent + b.sent = nil + q, err := b.Republish(ctx, "", "") + if err != nil || fmt.Sprint(q) != "[m2 0.1.0 m1 0.1.0]" || len(b.sent) != 2 { + t.Errorf("the tool with neither queued %v (%v), sent %d: want both versions, with no limit", q, err, len(b.sent)) + } + if _, err := b.Republish(ctx, "m1", "0.1.9"); err == nil { + t.Error("a version no reservation holds is not refused") + } + answer, err := AnswerQueue(Reader{Keys: func() ([]string, error) { return nil, nil }}, Reader{Keys: func() ([]string, error) { return b.acted.KeysOf(ctx, AllFilter) }, + Get: func(k string) (json.RawMessage, bool, error) { v, _, ok, err := b.acted.Get(ctx, k); return v, ok, err }}) + if err != nil || answer.Acted[ActedKey(subject)] != 1 { + t.Errorf("builder_publish_queue acted = %v (%v), want the running's sequence 1", answer.Acted, err) + } + }) +} diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index 89b0dc32..3a13dff1 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -54,13 +54,21 @@ type ReservedVersion struct { Repository string `json:"repository,omitempty"` Commit string `json:"commit,omitempty"` Tag string `json:"tag,omitempty"` + // Fingerprint is the fingerprint of the catalog entry the reservation was made from, so a publish again reads that + // entry (design 47 G A-76). + Fingerprint string `json:"fingerprint,omitempty"` + // Unstored are the artifacts a holder refused as not stored, with its words: said once, and not published again on + // events (design 47 G §588); builder_build_again builds the key again, which reserves anew. + Unstored map[string]string `json:"unstored,omitempty"` } // Source is what a build reserving a version is made from (design 47 G §370): the repository and commit the git tag is // written at, and whether the module has a Go package, whose version is tagged. type Source struct { Repository, Commit string - GoPackage bool + // Fingerprint is the fingerprint of the catalog entry the build is made from (design 47 G A-76). + Fingerprint string + GoPackage bool } // ReservedKey is the key of module's reservation of series (MAJOR.MINOR). @@ -135,7 +143,7 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts } } out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, next+1), Artifacts: map[string]string{}, At: r.now(), - Repository: src.Repository, Commit: src.Commit} + Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint} if src.GoPackage { out.Tag = GoTagOf(out.Version) } diff --git a/modules/mesh-delivery-builder/internal/versions/versions_test.go b/modules/mesh-delivery-builder/internal/versions/versions_test.go index 59b9b7a3..6ed783a7 100644 --- a/modules/mesh-delivery-builder/internal/versions/versions_test.go +++ b/modules/mesh-delivery-builder/internal/versions/versions_test.go @@ -73,6 +73,13 @@ func (m *mem) KeysOf(_ context.Context, filter string) ([]string, error) { out = append(out, k) continue } + if strings.Count(filter, ".") == 2 { // a three-token filter, `a.*.*` as the bus reads it + ft, kt := strings.Split(filter, "."), strings.Split(k, ".") + if len(kt) == 3 && kt[0] == ft[0] { + out = append(out, k) + } + continue + } if strings.HasPrefix(k, prefix) && !strings.Contains(k[len(prefix):], ".") { out = append(out, k) } @@ -540,39 +547,7 @@ func TestTheBuildersVersions(t *testing.T) { t.Error("80 % is not full") } }) - t.Run("publish again", func(t *testing.T) { - num := Entry{Name: "m/p", Kind: KindBundle, Version: "0.1.3", Fingerprint: "f1", Digest: "sha256:aa", State: Numbered} - ref := num - ref.State, ref.RefusedDigest = Refused, "sha256:aa" - other := ref - other.RefusedDigest = "sha256:bb" - pub := num - pub.State = Published - gone := pub - gone.State = Removed - tagged := num - tagged.Tag, tagged.Version = "feat-x", "0.1.3-feat-x.1" - for _, c := range []struct { - name string - entries []Entry - fp string - built bool - do string - }{ - {"a held build is said numbered again", []Entry{num}, "f1", true, AgainPublish}, - {"refused for its own bytes is published again", []Entry{ref}, "f1", true, AgainPublish}, - {"a gone file is built again under the same number", []Entry{ref}, "f1", false, AgainBuild}, - {"after a published it takes a new number", []Entry{pub}, "f1", true, AgainRenumber}, - {"after a removal it takes a new number", []Entry{gone}, "f1", false, AgainRenumber}, - {"refused for other bytes is refused", []Entry{other}, "f1", true, ""}, - {"a fingerprint not numbered is refused", []Entry{num}, "f2", true, ""}, - {"a tagged version is not a trunk fingerprint's", []Entry{tagged}, "f1", true, ""}, - } { - a, err := PublishAgain(c.entries, c.fp, c.built) - if (err != nil) != (c.do == "") || a.Do != c.do { - t.Errorf("%s: PublishAgain = %+v, %v; want %q", c.name, a, err, c.do) - } - } + t.Run("fresh claims and the floor", func(t *testing.T) { ctx := context.Background() // A fresh claim of a published fingerprint takes the next number; the old one stays. st := newMem() @@ -603,32 +578,8 @@ func TestTheBuildersVersions(t *testing.T) { if all, err := v.All(ctx); err != nil || len(all) != 2 { t.Errorf("All = %d entries, %v", len(all), err) } - // The hand acts: a held build is said numbered once more under a fresh dedup id; a gone one is built again; a - // published one takes a new number; a name with versions asks no floor. - if _, _, err := (Acts{Numberer: &Numberer{Versions: v}, Done: func(context.Context, string, string) (bool, error) { return true, nil }}).PublishAgain(ctx, "m/p", "f1", "w", "", time.Unix(4, 0)); err == nil { - t.Error("a version refused for other bytes is published again") - } _, _ = v.Mark(ctx, "m/p", "0.1.4", func(e *Entry) { e.State = Published }) - var rebuilt []string - done := true - acts := Acts{Numberer: &Numberer{Versions: v, Waits: &FloorWaits{}}, - Done: func(context.Context, string, string) (bool, error) { return done, nil }, - BuildAgain: func(_ context.Context, m, fp, _, _ string) (string, bool) { - rebuilt = append(rebuilt, m+" "+fp) - return "built again", false - }, - Reconsider: func(context.Context) {}} - // Nothing says a version numbered (design 47 G §349): a held build is refused before any number is taken. - if _, _, err := acts.PublishAgain(ctx, "m/p", "f1", "w", "me", time.Unix(5, 0)); err == nil || len(rebuilt) != 0 { - t.Errorf("publish again of a published, held build answered %v, rebuilt %v; want a refusal", err, rebuilt) - } - if es, _ := v.Of(ctx, "m/p"); len(es) != 2 { - t.Errorf("the refused publish again took a number: %+v", es) - } - done = false - if ag, _, err := acts.PublishAgain(ctx, "m/p", "f1", "w", "me", time.Unix(6, 0)); err != nil || ag.Do != AgainRenumber || ag.Entry.Version != "0.1.5" || len(rebuilt) != 1 { - t.Errorf("publish again of a gone build: %+v %v rebuilt %v", ag, err, rebuilt) - } + acts := Acts{Numberer: &Numberer{Versions: v, Waits: &FloorWaits{}}, Reconsider: func(context.Context) {}} if _, err := acts.FloorAgain(ctx, "m/p"); err == nil { t.Error("a name with versions asks its floor again") } diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 38debdc3..c25851da 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -113,7 +113,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if m.Module != "mesh-delivery-builder" { t.Errorf("module = %q", m.Module) } - if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory + if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory t.Errorf("state = %+v, want %+v", m.State, want) } // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and @@ -142,10 +142,10 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 // Design 51 *The floor*: the floor's three sources, the registry seats' versions, bridge_tags and indexer_requirements. // The watchdogs seat's presence says a node's machine is heard (issue 1966, ADR 0596 §2, ADR 0668 §3). if want := []string{"mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-forge-bridge.bridge_tags", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests", - "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.remove", "seat:container-image-registry.versions", + "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.publish", "seat:container-image-registry.remove", "seat:container-image-registry.versions", "seat:git.tag", - "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.remove", - "seat:npm-package-registry.versions", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + "seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-catalog.entry", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove", + "seat:npm-package-registry.versions", "seat:oci-artifact-registry.publish", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } // The builder hears the build seats' toolchain-reported and the node-engine seat's running (design 47 G §492, §546) and reads no @@ -172,7 +172,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 } // 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again. if want := []string{"builder_announce_again", "builder_ask_again", "builder_bounds", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views", - "builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_release", "builder_reserved", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) { + "builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserved", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §672: the toolchain waits' stream, kept 7 days within 16 MiB, and the bound's setting, 60 minutes. @@ -181,8 +181,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 m.Streams[1].Name != "mesh-delivery-builder-findings" || !reflect.DeepEqual(m.Streams[1].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) { t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, and the findings' stream", m.Streams) } - if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 1 { - t.Errorf("config = %+v, want toolchain-wait-minutes of 60", m.Config) + if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 2 || m.Config["republish-max-versions"].Default != float64(50) { + t.Errorf("config = %+v, want toolchain-wait-minutes of 60 and republish-max-versions of 50 (design 47 G §588, §633)", m.Config) } // Design 47 G §349: version.numbered is no longer said; old-form-closed on the family version, apart from the builds' ends. if want := []string{"build.asked-again", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) { @@ -222,7 +222,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1 t.Errorf("process = %v", p) } env, _ := p["env"].(map[string]any) - if len(env) != 2 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" { + if len(env) != 3 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_REPUBLISH_MAX_FILE"] != "${dir:state}/republish-max-versions" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" { t.Errorf("env = %v, want the broker file and the path of the toolchain wait's file, never the setting itself (design 47 G §672, A-71)", p["env"]) } for k, v := range env { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 8e65cc39..9577bc12 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -22,6 +22,7 @@ "builder_floor_waits", "builder_old_form", "builder_publish_again", + "builder_publish_queue", "builder_reserved", "builder_release", "builder_reserved_drop", @@ -82,6 +83,7 @@ "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", + "seat:mesh-catalog.entry", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", @@ -97,7 +99,11 @@ "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests", "mesh-delivery-indexer.indexer_read_modules", - "mesh-delivery-indexer.indexer_read_packages" + "mesh-delivery-indexer.indexer_read_packages", + "seat:container-image-registry.publish", + "seat:go-package-registry.publish", + "seat:npm-package-registry.publish", + "seat:oci-artifact-registry.publish" ], "state": [ { "name": "builds", "history": 5 }, @@ -112,6 +118,8 @@ { "name": "toolchains-heard", "history": 1 }, { "name": "waits", "history": 1, "max-mib": 4 }, { "name": "reserved", "history": 1 }, + { "name": "publish-queue", "history": 1, "max-mib": 4 }, + { "name": "acted", "history": 1, "max-mib": 1 }, { "name": "reasks", "history": 1 }, { "name": "versions", "history": 1 }, { "name": "first-views", "history": 1 } @@ -128,6 +136,14 @@ "maximum": 10080, "trusted": false, "default": 60 + }, + "republish-max-versions": { + "type": "number", + "description": "the most versions one event of a registry holder queues to publish again, newest first (design 47 G section 588); the rest wait for the next event or builder_publish_again.", + "minimum": 1, + "maximum": 10000, + "trusted": false, + "default": 50 } }, "logs": [ @@ -173,6 +189,14 @@ "mode": "0600", "content": "${config:toolchain-wait-minutes}" }, + { + "id": "republish-max", + "type": "file", + "path": "${dir:state}/republish-max-versions", + "owner": "mesh-delivery-builder", + "mode": "0600", + "content": "${config:republish-max-versions}" + }, { "id": "builder", "type": "process", @@ -183,7 +207,8 @@ ], "env": { "MESH_BROKER_FILE": "${dir:state}/broker", - "MESH_BUILDER_TOOLCHAIN_WAIT_FILE": "${dir:state}/toolchain-wait-minutes" + "MESH_BUILDER_TOOLCHAIN_WAIT_FILE": "${dir:state}/toolchain-wait-minutes", + "MESH_BUILDER_REPUBLISH_MAX_FILE": "${dir:state}/republish-max-versions" }, "user": "mesh-delivery-builder", "health": { From 381b55ef53041530fb8f3c495614e3cd39366e3a Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:16:40 +0100 Subject: [PATCH 06/10] BL17a: a key failed on a transport fault says it as 'transport fault: , , '; drops a committed binary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §568, A-76 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-delivery-builder/buildrun/asks.go | 9 ++-- .../buildrun/asks_test.go | 8 +-- .../mesh-delivery-builder/buildrun/faults.go | 53 +++++++++++++++++++ .../buildrun/faults_test.go | 23 ++++++++ .../mesh-delivery-builder/builds/builds.go | 12 +++-- .../builds/reference_test.go | 4 +- .../internal/build/claim_test.go | 2 +- .../internal/build/keyerror_test.go | 17 ++++++ .../internal/build/run.go | 15 ++++-- 9 files changed, 125 insertions(+), 18 deletions(-) create mode 100644 modules/mesh-delivery-builder/buildrun/faults.go create mode 100644 modules/mesh-delivery-builder/buildrun/faults_test.go create mode 100644 modules/mesh-delivery-builder/internal/build/keyerror_test.go diff --git a/modules/mesh-delivery-builder/buildrun/asks.go b/modules/mesh-delivery-builder/buildrun/asks.go index bbc793fb..7db40ad7 100644 --- a/modules/mesh-delivery-builder/buildrun/asks.go +++ b/modules/mesh-delivery-builder/buildrun/asks.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "log" - "strings" "sync" "time" @@ -106,7 +105,7 @@ var PutTry = 10 * time.Second // §481, §600: `accept..builder`). The put is a kept call (novox/hq ADR 0668 // §1, §2; issue 1966, issue 1748 note 27): it carries the ask's id as its message id, so the stream takes it once, and it // is made again through every transport fault (a deadline, no responders) within busretry.Bound, each try bounded by -// PutTry, until ctx ends. A put that still meets one is an error opening with buildstep.TransportFaultWords, so the key it +// PutTry, until ctx ends. A put that still meets one is an error opening `transport fault:` (TransportFault), so the key it // fails is one the builder does not take again by itself (ADR 0664 §3) and builder_build_again does. func (k Asker) Put(ctx context.Context, a buildstep.StepAsk) error { raw, err := buildstep.EncodeStepAsk(a) @@ -133,7 +132,8 @@ func (k Asker) Put(ctx context.Context, a buildstep.StepAsk) error { case err == nil: return nil case busretry.TimedOut(err) || busretry.Undelivered(err): - return fmt.Errorf("%sthe put of the %s ask %s: %w", buildstep.TransportFaultWords, a.Kind, a.ID, err) + seat, _ := buildstep.BuildSeat(a.Language) + return errors.New(TransportFault(FaultClass(err.Error()), seat, "no node", fmt.Sprintf("the put of the %s ask %s: %v", a.Kind, a.ID, err))) } return fmt.Errorf("putting the %s ask %s: %w", a.Kind, a.ID, err) } @@ -212,7 +212,8 @@ func (k Asker) Wait(ctx context.Context, a buildstep.StepAsk) (buildstep.Step, e if step.Passing && step.State == buildstep.StepFailed && (len(agains) > 0 || step.Run > 1) { // The step's own count of runs (ADR 0664 §2), not only the ones this wait saw: a builder that started again // reads the key's last value and none of the runs before it. - step.Reason = fmt.Sprintf(buildstep.TransportFaultWords+"%d runs (%s): %s", max(step.Run, len(agains)+1), strings.Join(append(agains, step.Node), ", "), step.Reason) + seat, _ := buildstep.BuildSeat(a.Language) + step.Reason = TransportFault(FaultClass(step.Reason), seat, step.Node, step.Reason) } return step, nil } diff --git a/modules/mesh-delivery-builder/buildrun/asks_test.go b/modules/mesh-delivery-builder/buildrun/asks_test.go index ed127a56..e31fd97e 100644 --- a/modules/mesh-delivery-builder/buildrun/asks_test.go +++ b/modules/mesh-delivery-builder/buildrun/asks_test.go @@ -72,11 +72,11 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { restore() switch { case c.transport: - if err == nil || !strings.HasPrefix(err.Error(), buildstep.TransportFaultWords) { + if err == nil || !strings.HasPrefix(err.Error(), TransportFaultPrefix+" deadline, node-build-go no node, ") { t.Errorf("%s: %v, want it opens with the transport fault's words", name, err) } case c.tries == 1: - if err == nil || p.tries != 1 || strings.Contains(err.Error(), buildstep.TransportFaultWords) { + if err == nil || p.tries != 1 || strings.HasPrefix(err.Error(), TransportFaultPrefix) { t.Errorf("%s: %v after %d tries, want one try and no transport fault", name, err, p.tries) } case err != nil || p.tries != c.tries || len(p.inner.Asks()) != 1 || len(p.ids) != c.tries || p.ids[0] != p.ids[len(p.ids)-1]: @@ -106,7 +106,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { } h.Write(st) } - if _, err := (Asker{Publisher: h, Waiter: h, Bound: time.Minute}).Ask(context.Background(), goAsk("mvl0frv18d1972")); err == nil || !strings.Contains(err.Error(), "a transport fault on 3 runs (ace, g14, shanks): lookup proxy: i/o timeout") { + if _, err := (Asker{Publisher: h, Waiter: h, Bound: time.Minute}).Ask(context.Background(), goAsk("mvl0frv18d1972")); err == nil || !strings.Contains(err.Error(), "transport fault: name lookup, node-build-go shanks, lookup proxy: i/o timeout") { t.Errorf("three runs then failed: %v", err) } h = &FakeAsks{} @@ -139,7 +139,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { // how many runs; a drop that failed is said, not discarded. late := &FakeAsks{} late.Write(buildstep.Step{ID: "mvl0frv18d1979", Node: "shanks", State: buildstep.StepFailed, Passing: true, Run: 3, Reason: "lookup proxy: i/o timeout"}) - if st, err := (Asker{Publisher: late, Waiter: late, Bound: time.Minute}).Wait(context.Background(), goAsk("mvl0frv18d1979")); err != nil || !strings.Contains(st.Reason, "a transport fault on 3 runs") { + if st, err := (Asker{Publisher: late, Waiter: late, Bound: time.Minute}).Wait(context.Background(), goAsk("mvl0frv18d1979")); err != nil || !strings.HasPrefix(st.Reason, "transport fault: name lookup, node-build-go shanks") { t.Errorf("a failed step of run 3 read alone says %q (%v), want it names 3 runs", st.Reason, err) } var said []string diff --git a/modules/mesh-delivery-builder/buildrun/faults.go b/modules/mesh-delivery-builder/buildrun/faults.go new file mode 100644 index 00000000..e1054451 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/faults.go @@ -0,0 +1,53 @@ +package buildrun + +import ( + "strings" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" +) + +// The words of a key that failed on a transport fault (design 47 G §568, A-76): `transport fault: , , +// `, the class one of ADR 0664 §1's four, the seat and node the step's, the cause the last run's in the build +// module's words. No other failure opens `transport fault:`. + +// TransportFaultPrefix opens the error of a key that failed on a transport fault. +const TransportFaultPrefix = builds.TransportFaultPrefix + +// The classes of a transport fault. +const ( + ClassNoResponders = "no responders" + ClassNameLookup = "name lookup" + ClassConnection = "connection" + ClassDeadline = "deadline" +) + +// FaultClass is the class of a call's error or a step's reason: the words the builder's retake reads a transport fault by +// (buildstep.PassingCause), read as no responders, a name lookup, a deadline or, for every other refused, reset or cut +// connection, a connection. +func FaultClass(cause string) string { + c := strings.ToLower(cause) + has := func(words ...string) bool { + for _, w := range words { + if strings.Contains(c, w) { + return true + } + } + return false + } + switch { + case has("no responders", strings.ToLower(buildstep.HolderSilentWords), "jetstream not enabled"): + return ClassNoResponders + case has("no such host", "name resolution", "lookup ", "could not resolve", "enotfound", "eai_again"): + return ClassNameLookup + case has("deadline", "timeout", "timed out", "etimedout"): + return ClassDeadline + } + return ClassConnection +} + +// TransportFault is the error of a key that failed on a transport fault of the seat's step on node, with the cause of its +// last run. +func TransportFault(class, seat, node, cause string) string { + return TransportFaultPrefix + " " + class + ", " + seat + " " + node + ", " + cause +} diff --git a/modules/mesh-delivery-builder/buildrun/faults_test.go b/modules/mesh-delivery-builder/buildrun/faults_test.go new file mode 100644 index 00000000..528b0699 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/faults_test.go @@ -0,0 +1,23 @@ +package buildrun + +import "testing" + +// Design 47 G §568, A-76: the class of a transport fault is read from the call's error, and its words are one form. +func TestFaultClassAndWords(t *testing.T) { + for cause, class := range map[string]string{ + "nats: no responders available for request": ClassNoResponders, + "the holder is silent: seat x answered": ClassNoResponders, + "lookup proxy: no such host": ClassNameLookup, + "Get https://r: context deadline exceeded": ClassDeadline, + "read tcp: i/o timeout": ClassDeadline, + "dial tcp: connection refused": ClassConnection, + "ssh: kex_exchange_identification: read: reset": ClassConnection, + } { + if got := FaultClass(cause); got != class { + t.Errorf("FaultClass(%q) = %q, want %q", cause, got, class) + } + } + if got, want := TransportFault(ClassDeadline, "node-build-go", "ace", "i/o timeout"), "transport fault: deadline, node-build-go ace, i/o timeout"; got != want { + t.Errorf("TransportFault = %q, want %q", got, want) + } +} diff --git a/modules/mesh-delivery-builder/builds/builds.go b/modules/mesh-delivery-builder/builds/builds.go index fe8d7f7b..f06ef70a 100644 --- a/modules/mesh-delivery-builder/builds/builds.go +++ b/modules/mesh-delivery-builder/builds/builds.go @@ -194,11 +194,15 @@ func FailedUntilChanged(err string) bool { return strings.HasPrefix(err, NotPinned) || strings.HasPrefix(err, VersionTaken) || strings.HasPrefix(err, RefusedKind) } -// FailedOnTransportFault says whether a failed key's error is a step's end after its third run met a transport fault -// (buildstep.TransportFaultWords): a fault of the network, not of the source, so the builder does not retake it by itself -// (ADR 0664 §3, which changes ADR 0429 clause 18 for this case); a person's hand act or a builder that starts again does. +// TransportFaultPrefix opens the error of a key that failed on a transport fault: `transport fault: , , +// ` (design 47 G §568, A-76). No other failure opens it. +const TransportFaultPrefix = "transport fault:" + +// FailedOnTransportFault says whether a failed key's error opens TransportFaultPrefix: a fault of the network, not of the +// source, so the builder does not retake it on its look (ADR 0664 §3, which ADR 0673 changes): a person's hand act, a builder +// that starts again, or an event that says its cause passed (design 47 G §568) builds it again. func FailedOnTransportFault(err string) bool { - return strings.Contains(err, buildstep.TransportFaultWords) + return strings.HasPrefix(err, TransportFaultPrefix) } // ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and diff --git a/modules/mesh-delivery-builder/builds/reference_test.go b/modules/mesh-delivery-builder/builds/reference_test.go index 1713338f..89b6d58a 100644 --- a/modules/mesh-delivery-builder/builds/reference_test.go +++ b/modules/mesh-delivery-builder/builds/reference_test.go @@ -103,13 +103,13 @@ func TestOnlyAPushFailureIsBuiltAgainBeforeTheFingerprintChanges(t *testing.T) { {VersionTaken + "@novox/mesh-sdk@0.1.15 holds another file", false}, {RefusedKind + "a package in go", false}, {"go test: 2 failed", true}, - {"a transport fault on 3 runs (ace, g14, novox)", true}, // ADR 0664: not "until changed" (a person's hand act builds it); the builder's own retake is FailedOnTransportFault's + {"transport fault: deadline, node-build-go ace, i/o timeout", true}, // ADR 0664: not "until changed" (a person's hand act builds it); the builder's own retake is FailedOnTransportFault's } { if got := FailedUntilChanged(c.err); got == c.again { t.Errorf("FailedUntilChanged(%q) = %v", c.err, got) } } - if !FailedOnTransportFault("mesh-build-go on ace: a transport fault on 3 runs (ace): i/o timeout") || FailedOnTransportFault("go test: 2 failed") || FailedOnTransportFault(NotStored+"code: 500") { + if !FailedOnTransportFault("transport fault: connection, node-build-go ace, connection refused") || FailedOnTransportFault("mesh-build-go on ace: a transport fault on 3 runs (ace)") || FailedOnTransportFault("go test: 2 failed") || FailedOnTransportFault(NotStored+"code: 500") { t.Error("FailedOnTransportFault must name the third run's end alone: the builder's own retake holds it (ADR 0664 §3)") } if NotStored != "not stored: " || NotPinned != "not pinned: " || VersionTaken != "version taken: " || RefusedKind != "refused kind: " { diff --git a/modules/mesh-delivery-builder/internal/build/claim_test.go b/modules/mesh-delivery-builder/internal/build/claim_test.go index eeda081f..c553addd 100644 --- a/modules/mesh-delivery-builder/internal/build/claim_test.go +++ b/modules/mesh-delivery-builder/internal/build/claim_test.go @@ -166,7 +166,7 @@ func TestAFailedKeyIsTakenAgainAsANewRevisionExceptARefusedKind(t *testing.T) { take bool }{{ClaimOptions{}, false}, {ClaimOptions{ByHand: true}, true}, {ClaimOptions{TakeOver: true}, true}} { st = newMem() - seed(t, st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.a", Error: "mesh-build-go on ace: a transport fault on 3 runs (ace, g14, novox): lookup proxy: i/o timeout"}) + seed(t, st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.a", Error: "transport fault: name lookup, node-build-go ace, lookup proxy: i/o timeout"}) if got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", o.opt); err != nil || (got != nil) != o.take { t.Errorf("a transport fault on 3 runs with %+v: claim = %v, %v, want taken %v", o.opt, got, err, o.take) } diff --git a/modules/mesh-delivery-builder/internal/build/keyerror_test.go b/modules/mesh-delivery-builder/internal/build/keyerror_test.go new file mode 100644 index 00000000..583d93ac --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/keyerror_test.go @@ -0,0 +1,17 @@ +package build + +import ( + "strings" + "testing" +) + +// Design 47 G §568: a key's error opens `transport fault:` only when every lane failed on one. +func TestKeyErrorOpensTransportFaultOnlyWhenAllLanesDid(t *testing.T) { + tf := "transport fault: deadline, node-build-go ace, i/o timeout" + if got := keyError([]string{tf, "mesh-build-node on g14: the step failed: tsc: 2 errors"}); strings.HasPrefix(got, "transport fault:") { + t.Errorf("beside a real failure the key opens %q", got) + } + if got := keyError([]string{tf, tf}); !strings.HasPrefix(got, "transport fault:") { + t.Errorf("two transport faults open %q", got) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index ff63d787..e03214a6 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -855,6 +855,12 @@ var keyPrefixes = []string{builds.NotPinned, builds.VersionTaken, builds.Refused // (` on : the step failed: not stored: ...`), the key's error opens with that prefix, so a reader of the // key need not search for it. func keyError(errs []string) string { + // A transport fault is the key's own words only when every lane failed on one: beside a real failure it is said after it, + // so the key is not built again on an event for a failure the source made (design 47 G §568). + errs = slices.Clone(errs) + slices.SortStableFunc(errs, func(a, b string) int { + return boolInt(strings.HasPrefix(a, builds.TransportFaultPrefix)) - boolInt(strings.HasPrefix(b, builds.TransportFaultPrefix)) + }) for _, prefix := range keyPrefixes { for i, e := range errs { if strings.HasPrefix(e, prefix) { @@ -991,11 +997,11 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes if err := asked.note(ctx, language, ask.ID, "asked "+l.module); err != nil { // The note and the put are one act (novox/hq ADR 0668 §1; issue 1748 note 27): a note a transport fault kept from // the key fails the key with that fault's words, which builder_build_again takes (ADR 0664 §3). - fault := "" + l.err = fmt.Errorf("the ask of %s could not be written to the key: %w", l.module, err) if busretry.TimedOut(err) || busretry.Undelivered(err) { - fault = sdkstep.TransportFaultWords + "the note of the ask: " + seat, _ := sdkstep.BuildSeat(language) + l.err = errors.New(buildrun.TransportFault(buildrun.FaultClass(err.Error()), seat, "no node", l.err.Error())) } - l.err = fmt.Errorf("%sthe ask of %s could not be written to the key: %w", fault, l.module, err) return } if err := b.Asks.Put(ctx, ask); err != nil { @@ -1017,6 +1023,9 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes } if step.State != sdkstep.StepPassed { l.err = fmt.Errorf("%s on %s: the ask %s %s: %s", l.module, step.Node, ask.ID, step.State, step.Reason) + if strings.HasPrefix(step.Reason, builds.TransportFaultPrefix) { + l.err = errors.New(step.Reason) // a key's error opens with it (design 47 G §568) + } return } var answer sdkstep.BuildAnswer From 2339eb60b8aa545852404239e062e7ed37dad9f9 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:19:08 +0100 Subject: [PATCH 07/10] BL17b: a key failed on a transport fault is built again on the events that say its cause passed, counted in event-builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §568 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-delivery-builder/builds/builds.go | 20 ++ .../cmd/mesh-delivery-builder/main.go | 4 + .../cmd/mesh-delivery-builder/retake.go | 31 +++ .../internal/build/claim.go | 12 +- .../internal/build/retake.go | 226 ++++++++++++++++++ .../internal/build/retake_test.go | 130 ++++++++++ .../internal/build/run.go | 10 +- 7 files changed, 429 insertions(+), 4 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/retake.go create mode 100644 modules/mesh-delivery-builder/internal/build/retake.go create mode 100644 modules/mesh-delivery-builder/internal/build/retake_test.go diff --git a/modules/mesh-delivery-builder/builds/builds.go b/modules/mesh-delivery-builder/builds/builds.go index f06ef70a..0f1b118e 100644 --- a/modules/mesh-delivery-builder/builds/builds.go +++ b/modules/mesh-delivery-builder/builds/builds.go @@ -245,6 +245,11 @@ type BuilderBuild struct { // oci-artifact-registry (design 47 G §671): every artifact of the build is kept there, each with its digest in // Artifacts. A build not kept names none; the builder carries it onto the build's end (BuildLogEntry.KeptOn). KeptOn string `json:"kept-on,omitempty"` + // EventBuilds counts, in the key's newest revision, the revisions an event started since the last one a person started + // (design 47 G §568, ADR 0673): the builder builds a key that failed on a transport fault again on an event for the third + // time at most, then asks the operator. A person's build (builder_build_again, the operator's answer) starts the count + // again. A reader that does not know the field reads none. + EventBuilds int `json:"event-builds,omitempty"` } func (b BuilderBuild) check() error { @@ -365,3 +370,18 @@ func DecodeBuilderBuild(data []byte) (BuilderBuild, error) { // SameBuilderBuild says whether two builds are the same value, so the builder writes only when they differ. func SameBuilderBuild(a, b BuilderBuild) bool { return reflect.DeepEqual(a, b) } + +// TransportFaultOf reads a key's error `transport fault: , , ` (design 47 G §568, A-76): ok is false +// for an error that does not open so. The node of a fault no node ran (the put of an ask) is `no node`. +func TransportFaultOf(err string) (class, seat, node, cause string, ok bool) { + rest, found := strings.CutPrefix(err, TransportFaultPrefix+" ") + if !found { + return "", "", "", "", false + } + parts := strings.SplitN(rest, ", ", 3) + if len(parts) != 3 { + return "", "", "", "", false + } + seat, node, _ = strings.Cut(parts[1], " ") + return parts[0], seat, node, parts[2], seat != "" && node != "" +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 4122bc39..d1496865 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -233,6 +233,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Unreleased: reserved.unreleasedWords, Max: republishMax, Logf: logger.Printf} acts.Republish = republish.Republish again := &republishing{rp: republish} + retake := &retaking{rt: &build.Retaker{Builder: b, Acted: awaitedReserved{awaitedVersions{b: actedKV, bucket: versions.ActedBucket}}, Entry: entryVia(calls.Call), Logf: logger.Printf}} queueKV.Ready(again.bound(ctx, js, logger.Printf)) actedKV.Ready(again.bound(ctx, js, logger.Printf)) // A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base @@ -379,6 +380,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if again.heard(ctx, m, logger.Printf) { // the publish again's own subjects, left for the bus to give again when its state cannot take them return } + if retake.heard(ctx, m, logger.Printf) { // a key failed on a transport fault, built again when its cause passed (§568) + return + } if reads.heard(m, logger.Printf) || heardTools.Reported(ctx, m) || heardTools.Running(ctx, m) || floor.heard(ctx, m) || reserved.heard(ctx, m) || seats.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4) return } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/retake.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/retake.go new file mode 100644 index 00000000..90215ec6 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/retake.go @@ -0,0 +1,31 @@ +package main + +import ( + "context" + + "github.com/nats-io/nats.go/jetstream" + + "mesh-delivery-builder/internal/build" +) + +// The builder's retake of a key failed on a transport fault, on the node-engine seat's events (design 47 G §568, ADR 0673): the +// wiring of build.Retaker to the events consumer. +type retaking struct{ rt *build.Retaker } + +// heard hands a message of the events consumer to the retake: one of its subjects whose state cannot take it is left for the bus to +// give again (Nak); every message goes on to the other parts. +func (g *retaking) heard(ctx context.Context, m jetstream.Msg, logf func(string, ...any)) (stop bool) { + meta, err := m.Metadata() + if err != nil { + return false + } + handled, err := g.rt.Heard(ctx, m.Subject(), m.Data(), meta.Sequence.Stream) + if handled && err != nil { + logf("%v; heard again", err) + if nakErr := m.Nak(); nakErr != nil { + logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr) + } + return true + } + return false +} diff --git a/modules/mesh-delivery-builder/internal/build/claim.go b/modules/mesh-delivery-builder/internal/build/claim.go index ea041b46..5408e604 100644 --- a/modules/mesh-delivery-builder/internal/build/claim.go +++ b/modules/mesh-delivery-builder/internal/build/claim.go @@ -54,6 +54,9 @@ type ClaimOptions struct { // ByHand is a person's hand act (BuildAgain): a key failed on a transport fault is taken again only by it, or by a builder // that has just started (TakeOver), never by the builder's own retake (ADR 0664 §3). ByHand bool + // ByEvent is the builder's retake on an event that says a transport fault's cause passed (design 47 G §568): it takes a key + // failed on a transport fault, as a new revision that counts one more of EventBuilds. + ByEvent bool // ImageBuilder says whether a node runs a healthy image build module now: a key failed for NoImageBuilderPrefix is taken // again only then. Nil means none does. ImageBuilder func() bool @@ -213,11 +216,18 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string return nil, nil case cur.State == builds.BuildFailed && (builds.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)): return nil, nil - case cur.State == builds.BuildFailed && builds.FailedOnTransportFault(cur.Error) && !opt.ByHand && !opt.TakeOver: + case cur.State == builds.BuildFailed && builds.FailedOnTransportFault(cur.Error) && !opt.ByHand && !opt.TakeOver && !opt.ByEvent: return nil, nil case cur.State == builds.BuildFailed && strings.HasPrefix(cur.Error, NoImageBuilderPrefix) && (opt.ImageBuilder == nil || !opt.ImageBuilder()): return nil, nil case cur.State == builds.BuildFailed: + building.EventBuilds = cur.EventBuilds // a person's build starts the count again, an event's adds one (design 47 G §568) + switch { + case opt.ByHand: + building.EventBuilds = 0 + case opt.ByEvent: + building.EventBuilds++ + } return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) }) case cur.State == builds.BuildQueued && cur.Waits != "": if opt.Published == nil || !opt.Published(cur.Waits) { diff --git a/modules/mesh-delivery-builder/internal/build/retake.go b/modules/mesh-delivery-builder/internal/build/retake.go new file mode 100644 index 00000000..09ae2f4d --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/retake.go @@ -0,0 +1,226 @@ +package build + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "mesh-delivery-builder/internal/versions" +) + +// **A key that failed on a transport fault is built again on an event that says its cause passed** (design 47 G §568, ADR 0673 +// §1 to §4, which changes ADR 0664 §3): the node-engine seat's `running` of a module whose catalog graph entry claims the build +// seat the failed step ran on, on any node, and its `manifest-reads-changed` of the node the step ran on with `why` `returned`. +// Each message is acted on once, by its stream sequence (state `acted`). A key is built again once per event and only while its +// newest revision is that failure, as a new revision counting one more of EventBuilds; after MostEventBuilds the next event +// starts nothing and asks the operator once. + +// MostEventBuilds is how many revisions events start, since the last a person started, before the operator is asked. +const MostEventBuilds = 3 + +// Retake builds the failed key again on an event: it starts the look of the key's module at its fingerprint, which claims the +// key as an event's (ClaimOptions.ByEvent). started is false when the key is not failed on a transport fault, when the module +// has another fingerprint now, or when a look holds the key; ask is true when MostEventBuilds are spent, with the key as it +// stands and its revision. +func (b *Builder) Retake(ctx context.Context, key string) (started, ask bool, cur builds.BuilderBuild, rev uint64, err error) { + b.init() + module, fp, err := builds.ParseBuilderBuildKey(key) + if err != nil { + return false, false, cur, 0, err + } + value, rev, found, err := b.Claimer.Store.Get(ctx, key) + if err != nil || !found { + return false, false, cur, 0, err + } + if cur, err = builds.DecodeBuilderBuild(value); err != nil { + return false, false, cur, rev, fmt.Errorf("%s holds what is not a build: %w", key, err) + } + if cur.State != builds.BuildFailed || !builds.FailedOnTransportFault(cur.Error) { + return false, false, cur, rev, nil + } + if cur.EventBuilds >= MostEventBuilds { + return false, true, cur, rev, nil + } + b.mu.Lock() + m, known := b.known[module] + if !known || m.Fp2 != fp || m.Manifest == "" || b.inflight[key] || b.hands[key] != nil { + b.mu.Unlock() + return false, false, cur, rev, nil + } + b.inflight[key], b.events[key] = true, true + b.mu.Unlock() + b.logf("%s: failed (%s); built again on an event, build %d of %d since a person's", key, cur.Error, cur.EventBuilds+1, MostEventBuilds) + b.startOnce(ctx, key, module, m) + return true, false, cur, rev, nil +} + +// Retaker hears the node-engine seat's events and builds again what they say can pass (Builder.Retake). +type Retaker struct { + Builder *Builder + // Acted is the state acted: the last stream sequence acted on for each subject, and the revision the operator was asked at. + Acted versions.ReserveStore + // Entry reads the catalog graph entry of a fingerprint (the seat mesh-catalog's verb entry). + Entry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) + // Ask asks the operator about a key whose event builds are spent (the seat operator-channel's ask). + Ask func(ctx context.Context, key string, cur builds.BuilderBuild) error + Logf func(string, ...any) +} + +func (r *Retaker) logf(f string, a ...any) { + if r.Logf != nil { + r.Logf(f, a...) + } +} + +// retakeKey is the acted key of a subject's sequence, apart from the publish again's. +func retakeKey(subject string) string { return versions.ActedKey("retake:" + subject) } + +// askedKey is the acted key of the revision of a build key the operator was asked at. +func askedKey(key string) string { return versions.ActedKey("retake-asked:" + key) } + +// Heard takes one message of the node-engine seat's running or manifest-reads-changed in, at stream sequence seq, and acts on +// it once. A subject that is neither is not its; an error leaves it not acted on, so it is read again. +func (r *Retaker) Heard(ctx context.Context, subject string, data []byte, seq uint64) (bool, error) { + running := strings.HasPrefix(subject, strings.TrimSuffix(nodeengine.SubjectRunningAll, "*.*")) + changed := strings.HasPrefix(subject, "mesh.seat.node-engine.event."+nodeengine.EventManifestReadsChanged+".") + if !running && !changed { + return false, nil + } + key := retakeKey(subject) + raw, rev, found, err := r.Acted.Get(ctx, key) + if err != nil { + return true, fmt.Errorf("the acted sequence of %s cannot be read: %w", subject, err) + } + if found { + var a versions.Acted + if json.Unmarshal(raw, &a) == nil && a.Sequence >= seq { + return true, nil + } + } + var want func(seat, node string) bool + if running { + want, err = r.runningWant(ctx, data) + } else { + want, err = r.returnedWant(data) + } + if err == nil && want != nil { + err = r.retakeAll(ctx, want) + } + if err != nil { + return true, err + } + val, _ := json.Marshal(versions.Acted{Sequence: seq}) + if found { + _, err = r.Acted.Update(ctx, key, val, rev) + } else { + _, err = r.Acted.Create(ctx, key, val) + } + if err != nil && !errors.Is(err, versions.ErrConflict) { + return true, fmt.Errorf("the acted sequence of %s cannot be kept: %w", subject, err) + } + return true, nil +} + +// runningWant is what a running says passed: the build seats the running module's catalog graph entry claims, on any node. +func (r *Retaker) runningWant(ctx context.Context, data []byte) (func(seat, node string) bool, error) { + var ev nodeengine.Running + if err := json.Unmarshal(data, &ev); err != nil || ev.Module == "" || ev.Node == "" { + r.logf("a running that does not read is not acted on: %v", err) + return nil, nil + } + if ev.Stopped || ev.Fingerprint == "" { + return nil, nil + } + e, err := r.Entry(ctx, ev.Fingerprint) + if err != nil { + return nil, fmt.Errorf("the catalog entry %s of %s cannot be read: %w", ev.Fingerprint, ev.Module, err) + } + var seats []string + for _, s := range e.Serves { + if seat, _, _ := strings.Cut(s, "."); strings.HasPrefix(seat, "node-build-") && !slices.Contains(seats, seat) { + seats = append(seats, seat) + } + } + if len(seats) == 0 { + return nil, nil + } + return func(seat, _ string) bool { return slices.Contains(seats, seat) }, nil +} + +// returnedWant is what a manifest-reads-changed says passed: only one with why returned, for the node it names. +func (r *Retaker) returnedWant(data []byte) (func(seat, node string) bool, error) { + var ev nodeengine.ManifestReadsChanged + if err := json.Unmarshal(data, &ev); err != nil || ev.Node == "" { + r.logf("a manifest-reads-changed that does not read is not acted on: %v", err) + return nil, nil + } + if ev.Why != nodeengine.ManifestReadsChangedWhyReturned { + return nil, nil + } + return func(_, node string) bool { return node == ev.Node }, nil +} + +// retakeAll builds again every key failed on a transport fault whose seat and node want says passed. +func (r *Retaker) retakeAll(ctx context.Context, want func(seat, node string) bool) error { + keys, err := r.Builder.Claimer.Store.Keys(ctx) + if err != nil { + return fmt.Errorf("the build keys cannot be listed: %w", err) + } + for _, key := range keys { + raw, _, found, err := r.Builder.Claimer.Store.Get(ctx, key) + if err != nil || !found { + continue + } + v, err := builds.DecodeBuilderBuild(raw) + if err != nil || v.State != builds.BuildFailed { + continue + } + _, seat, node, _, ok := builds.TransportFaultOf(v.Error) + if !ok || !want(seat, node) { + continue + } + started, ask, cur, rev, err := r.Builder.Retake(ctx, key) + if err != nil { + return err + } + if started || !ask || r.Ask == nil { + continue + } + if err := r.askOnce(ctx, key, cur, rev); err != nil { + return err + } + } + return nil +} + +// askOnce asks the operator about key once per revision of the key. +func (r *Retaker) askOnce(ctx context.Context, key string, cur builds.BuilderBuild, rev uint64) error { + ak := askedKey(key) + raw, arev, found, err := r.Acted.Get(ctx, ak) + if err != nil { + return err + } + val := []byte(fmt.Sprint(rev)) + if found && string(raw) == string(val) { + return nil + } + if err := r.Ask(ctx, key, cur); err != nil { + return fmt.Errorf("the operator cannot be asked about %s: %w", key, err) + } + if found { + _, err = r.Acted.Update(ctx, ak, val, arev) + } else { + _, err = r.Acted.Create(ctx, ak, val) + } + if err != nil && !errors.Is(err, versions.ErrConflict) { + return err + } + return nil +} diff --git a/modules/mesh-delivery-builder/internal/build/retake_test.go b/modules/mesh-delivery-builder/internal/build/retake_test.go new file mode 100644 index 00000000..0999b450 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/retake_test.go @@ -0,0 +1,130 @@ +package build + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "testing" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" +) + +type actedStore struct { + mu sync.Mutex + vals map[string][]byte + revs map[string]uint64 +} + +func (s *actedStore) Get(_ context.Context, k string) ([]byte, uint64, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.vals[k] + return v, s.revs[k], ok, nil +} +func (s *actedStore) Create(_ context.Context, k string, v []byte) (uint64, error) { + return s.Update(context.Background(), k, v, 0) +} +func (s *actedStore) Update(_ context.Context, k string, v []byte, _ uint64) (uint64, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.vals == nil { + s.vals, s.revs = map[string][]byte{}, map[string]uint64{} + } + s.vals[k], s.revs[k] = v, s.revs[k]+1 + return s.revs[k], nil +} +func (s *actedStore) KeysOf(context.Context, string) ([]string, error) { return nil, nil } +func (s *actedStore) Delete(_ context.Context, k string) error { delete(s.vals, k); return nil } + +// Design 47 G §568, ADR 0673: a key failed on a transport fault is built again on the node-engine seat's events that say its +// cause passed, once per event, counted in event-builds; after three the next event asks the operator once. +func TestRetakeOnAnEvent(t *testing.T) { + r := newRig(t, goBundle) + // The first look creates the key and its note meets the fault; a retake's claim is an update that succeeds and the note after + // it is the second. + fault := fmt.Errorf("the key: %w", context.DeadlineExceeded) + failOnce := func() { r.st.before = func() { r.st.before = func() { r.st.faults = []error{fault} } } } + r.st.faults = []error{fault} + if v := r.consider(t, "modules/m"); !builds.FailedOnTransportFault(v.Error) { + t.Fatalf("the key = %+v, want failed on a transport fault", v) + } + var asked []string + rt := &Retaker{Builder: r.b, Acted: &actedStore{}, + Entry: func(context.Context, string) (meshcatalog.CatalogEntry, error) { + return meshcatalog.CatalogEntry{Serves: []string{"node-build-go.steps"}}, nil + }, + Ask: func(_ context.Context, key string, _ builds.BuilderBuild) error { + asked = append(asked, key) + return nil + }} + running, _ := json.Marshal(nodeengine.Running{Module: "mesh-build-go", Node: "g14", Fingerprint: "f1"}) + started, _ := json.Marshal(nodeengine.ManifestReadsChanged{Node: "ace", Why: "started"}) + heard := func(seq uint64, subject string, data []byte) { + t.Helper() + if ok, err := rt.Heard(context.Background(), subject, data, seq); !ok || err != nil { + t.Fatalf("heard %s: %v, %v", subject, ok, err) + } + r.b.Wait() + } + runningSubject := "mesh.seat.node-engine.event.running.g14.mesh-build-go" + readsSubject := "mesh.seat.node-engine.event.manifest-reads-changed.ace" + // A `started` builds nothing, and the same message again builds nothing. + heard(1, readsSubject, started) + if v := valueOf(t, r.st, r.key); v.EventBuilds != 0 || v.State != builds.BuildFailed { + t.Fatalf("a started: the key = %+v", v) + } + for i, ev := range []struct { + seq uint64 + subject string + data []byte + }{{2, runningSubject, running}, {3, runningSubject + ".2", running}, {4, runningSubject + ".3", running}} { + failOnce() + heard(ev.seq, ev.subject, ev.data) + if v := valueOf(t, r.st, r.key); v.EventBuilds != i+1 || !builds.FailedOnTransportFault(v.Error) { + t.Fatalf("event %d: the key = %+v, want event-builds %d failed on a transport fault", i+1, v, i+1) + } + } + heard(4, runningSubject+".3", running) // a sequence heard is not acted on again + if v := valueOf(t, r.st, r.key); v.EventBuilds != 3 || len(asked) != 0 { + t.Fatalf("a sequence heard again: the key = %+v, asked %v", v, asked) + } + heard(5, runningSubject, running) + heard(6, runningSubject, running) + if v := valueOf(t, r.st, r.key); v.EventBuilds != 3 || len(asked) != 1 || asked[0] != r.key { + t.Fatalf("the fourth event: the key = %+v, asked %v, want nothing built and the operator asked once", v, asked) + } + // A person's build starts the count again. + if res := r.b.BuildAgain(context.Background(), "m", fp1, "Build once more", "operator", 2e9); res.Refused != "" { + t.Fatalf("built again by hand: %+v", res) + } + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.EventBuilds != 0 { + t.Fatalf("after a person's build: the key = %+v, want event-builds 0", v) + } +} + +// A manifest-reads-changed with why returned builds again the keys whose step ran on its node, and no other's. +func TestRetakeOnANodeReturned(t *testing.T) { + r := newRig(t, goBundle) + r.consider(t, "modules/m") + seed(t, r.st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.builder", Manifest: mdigest('a'), + Error: "transport fault: connection, node-build-go ace, dial tcp: connection refused"}) + rt := &Retaker{Builder: r.b, Acted: &actedStore{}} + for seq, c := range []struct { + node, why string + built bool + }{{"g14", "returned", false}, {"ace", "started", false}, {"ace", "returned", true}} { + data, _ := json.Marshal(nodeengine.ManifestReadsChanged{Node: c.node, Why: c.why}) + if _, err := rt.Heard(context.Background(), "mesh.seat.node-engine.event.manifest-reads-changed."+c.node, data, uint64(seq+1)); err != nil { + t.Fatal(err) + } + r.b.Wait() + if v := valueOf(t, r.st, r.key); (v.State == builds.BuildDone) != c.built { + t.Fatalf("%s %s: the key = %+v, want built %v", c.node, c.why, v, c.built) + } + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index e03214a6..cc10b0d1 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -132,6 +132,8 @@ type Builder struct { waiting map[string]map[string]chan string // hands are the requests to build a key again by hand that wait for the claim and its asks (BuildAgain), by build key. hands map[string]*handWatch + // events are the keys an event asks the builder to build again (Retake), until their look claims them. + events map[string]bool // released are the keys builder_release is building now, whatever toolchain their fingerprint names (§276, §672). released map[string]bool // builds are the ends that withdraw the asks of a key's build in flight, by build key (ADR 0664 §4): a newer fingerprint of @@ -162,7 +164,7 @@ func (b *Builder) init() { b.cands, b.via, b.trunkFP = map[string]meshcatalog.CandidatesByGroupValue{}, map[string]via{}, map[string]bool{} b.known, b.inflight, b.again, b.plans, b.tried = map[string]meshcatalog.ModulesByModuleValue{}, map[string]bool{}, map[string]meshcatalog.ModulesByModuleValue{}, map[string]planned{}, map[string]bool{} b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{} - b.hands = map[string]*handWatch{} + b.hands, b.events = map[string]*handWatch{}, map[string]bool{} b.builds = map[string]context.CancelCauseFunc{} }) } @@ -493,6 +495,8 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB first := !b.tried[key] b.tried[key] = true byHand := b.hands[key] != nil + byEvent := b.events[key] + delete(b.events, key) b.mu.Unlock() var numbering versions.Numbering if trunk && planErr == nil && !nothing { @@ -507,7 +511,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB } // A key built again by hand is taken over as it stands too: one left building by a look that ended without its write // (issue 1966) is the hand tool's to finish. - cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first || byHand, ByHand: byHand, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished, + cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first || byHand, ByHand: byHand, ByEvent: byEvent, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished, Stored: b.storedOnce(key)}) if err != nil { b.logf("%s: %v", module, err) @@ -576,7 +580,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB b.logf("%s: left building, the toolchains moved under it", cl.Key) return } - result.By, result.Manifest = b.Claimer.By, m.Manifest + result.By, result.Manifest, result.EventBuilds = b.Claimer.By, m.Manifest, cl.Build.EventBuilds if len(cl.Gone) > 0 && result.State == builds.BuildDone { result.Step += " (built again: the artifact store no longer held " + strings.Join(cl.Gone, ", ") + ")" } From 50cca4309fb069f488824ed4285d943158b157e5 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:25:00 +0100 Subject: [PATCH 08/10] BL17c: the operator is asked once a key's event builds are spent; Build once more builds it as a person's build, Leave it nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The builder's asker on the seat operator-channel (level approve, about the key, state asked), the teller's three bus adapters (publish, the router's record through the seat's holder on the seat list, decided heard through the hub's events handler); the manifest uses the seat and consumes decided. Builds: design 47 G §568, A-74 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-builder/main.go | 16 +- .../cmd/mesh-delivery-builder/retake.go | 93 ++++++++++ modules/mesh-delivery-builder/go.mod | 2 + .../internal/build/retake.go | 4 +- .../internal/build/retake_test.go | 2 +- .../internal/build/retakeask.go | 162 ++++++++++++++++++ .../internal/build/retakeask_test.go | 113 ++++++++++++ .../mesh-delivery-builder/manifest_test.go | 10 +- modules/mesh-delivery-builder/module.json | 6 +- 9 files changed, 398 insertions(+), 10 deletions(-) create mode 100644 modules/mesh-delivery-builder/internal/build/retakeask.go create mode 100644 modules/mesh-delivery-builder/internal/build/retakeask_test.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index d1496865..0b856d1a 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -24,6 +24,8 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" @@ -233,7 +235,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Unreleased: reserved.unreleasedWords, Max: republishMax, Logf: logger.Printf} acts.Republish = republish.Republish again := &republishing{rp: republish} - retake := &retaking{rt: &build.Retaker{Builder: b, Acted: awaitedReserved{awaitedVersions{b: actedKV, bucket: versions.ActedBucket}}, Entry: entryVia(calls.Call), Logf: logger.Printf}} + // The operator is asked once a key's event builds are spent (design 47 G §568): the builder's asker on the seat operator-channel. + decided := &decisions{subject: oc.SubjectDecided(cred.Module)} + retakeActed := awaitedReserved{awaitedVersions{b: actedKV, bucket: versions.ActedBucket}} + operator := &build.OperatorAsk{Builder: b, Acted: retakeActed, Ctx: ctx, Logf: logger.Printf, + Client: &occ.Client{Module: cred.Module, State: calls.State("asked"), Publish: publishVia(ctx, js), + Record: recordVia(calls.Call, calls.Ask), Subscribe: decided.hear(logger.Printf)}} + if err := operator.Listen(); err != nil { + return fmt.Errorf("hearing the operator's answers: %w", err) + } + retake := &retaking{rt: &build.Retaker{Builder: b, Acted: retakeActed, Entry: entryVia(calls.Call), Ask: operator.Ask, Logf: logger.Printf}} queueKV.Ready(again.bound(ctx, js, logger.Printf)) actedKV.Ready(again.bound(ctx, js, logger.Printf)) // A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base @@ -380,6 +391,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if again.heard(ctx, m, logger.Printf) { // the publish again's own subjects, left for the bus to give again when its state cannot take them return } + if decided.heard(m) { // the operator's answer to an ask about a key (§568) + return + } if retake.heard(ctx, m, logger.Printf) { // a key failed on a transport fault, built again when its cause passed (§568) return } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/retake.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/retake.go index 90215ec6..90db6669 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/retake.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/retake.go @@ -2,7 +2,12 @@ package main import ( "context" + "encoding/json" + "fmt" + mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + stdio "git.novox.be/novox/mesh-sdk/go" "github.com/nats-io/nats.go/jetstream" "mesh-delivery-builder/internal/build" @@ -29,3 +34,91 @@ func (g *retaking) heard(ctx context.Context, m jetstream.Msg, logf func(string, } return false } + +// The builder's three bus adapters for the operator-channel client (design 47 G §568), as the teller has them: its own ask +// published, the router's record read through the seat, its own `decided` heard through the module's hub. + +// publishVia publishes an accept of the seat, de-duplicated by id: the same id twice is one message. +func publishVia(ctx context.Context, js jetstream.JetStream) func(subject string, body any, id string) error { + return func(subject string, body any, id string) error { + data, err := json.Marshal(body) + if err != nil { + return err + } + var opts []jetstream.PublishOpt + if id != "" { + opts = append(opts, jetstream.WithMsgID(id)) + } + _, err = js.Publish(ctx, subject, data, opts...) + return err + } +} + +// routerBucket is the bucket of the router's asks: the state `asks` of the seat operator-channel (the contract's StateAsks) in +// the bucket of the seat's holder on the seat list, which the seat mesh-module-assigner's verb assignments answers with seats +// (design 47 G A-74); no module's name is in the program. +func routerBucket(call func(key string, body any) (json.RawMessage, error)) (string, error) { + raw, err := call("seat:"+mma.Seat+"."+mma.VerbAssignments, mma.AssignmentsInput{Seats: mma.AssignmentsInputSeatsTrue}) + if err != nil { + return "", fmt.Errorf("who holds the seat %s is not asked of the seat %s: %w", oc.Seat, mma.Seat, err) + } + var held struct { + Seats []mma.SeatHolders `json:"seats"` + } + if err := json.Unmarshal(raw, &held); err != nil { + return "", fmt.Errorf("the answer of %s.%s is not its seat's: %w", mma.Seat, mma.VerbAssignments, err) + } + for _, s := range held.Seats { + if s.Seat == oc.Seat && len(s.Holders) > 0 { + return s.Holders[0].Module + "_" + oc.StateAsks, nil + } + } + return "", fmt.Errorf("no module holds the seat %s on the seat list, so the router's record of an ask is not read", oc.Seat) +} + +// recordVia reads the router's record of one of this module's asks: a get of `.` in the bucket the seat's holder +// keeps (record is process.Conn.Ask). +func recordVia(call func(key string, body any) (json.RawMessage, error), record func(bucket, id string) (*stdio.StateEntry, error)) func(id string) (*stdio.StateEntry, error) { + return func(id string) (*stdio.StateEntry, error) { + bucket, err := routerBucket(call) + if err != nil { + return nil, err + } + return record(bucket, id) + } +} + +// decisions holds the handler of the router's `decided` event for this module: the module's one hub has the pattern +// `mesh.seat.*.event.>`, which covers it, so the events handler hands the message here (a hub refuses two patterns that overlap). +type decisions struct { + subject string + handle func(jetstream.Msg) +} + +// hear is the client's Subscribe: it keeps the handler of the warrants the router says to this module. +func (d *decisions) hear(logf func(string, ...any)) func(pattern string, handle func(stdio.Envelope) error) error { + return func(pattern string, handle func(stdio.Envelope) error) error { + d.handle = func(m jetstream.Msg) { + if err := handle(stdio.Envelope{Key: pattern, Body: m.Data()}); err != nil { + logf("a warrant could not be handled and is given again: %v", err) + if nakErr := m.Nak(); nakErr != nil { + logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr) + } + return + } + if err := m.Ack(); err != nil { + logf("a warrant was handled but could not be acknowledged: %v", err) + } + } + return nil + } +} + +// heard takes the message when it is this module's `decided`. +func (d *decisions) heard(m jetstream.Msg) bool { + if d.handle == nil || m.Subject() != d.subject { + return false + } + d.handle(m) + return true +} diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index cf46a190..18e70a23 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -9,6 +9,8 @@ require ( git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/git v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-builder/internal/build/retake.go b/modules/mesh-delivery-builder/internal/build/retake.go index 09ae2f4d..22dd339c 100644 --- a/modules/mesh-delivery-builder/internal/build/retake.go +++ b/modules/mesh-delivery-builder/internal/build/retake.go @@ -69,7 +69,7 @@ type Retaker struct { // Entry reads the catalog graph entry of a fingerprint (the seat mesh-catalog's verb entry). Entry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) // Ask asks the operator about a key whose event builds are spent (the seat operator-channel's ask). - Ask func(ctx context.Context, key string, cur builds.BuilderBuild) error + Ask func(ctx context.Context, key string, cur builds.BuilderBuild, rev uint64) error Logf func(string, ...any) } @@ -211,7 +211,7 @@ func (r *Retaker) askOnce(ctx context.Context, key string, cur builds.BuilderBui if found && string(raw) == string(val) { return nil } - if err := r.Ask(ctx, key, cur); err != nil { + if err := r.Ask(ctx, key, cur, rev); err != nil { return fmt.Errorf("the operator cannot be asked about %s: %w", key, err) } if found { diff --git a/modules/mesh-delivery-builder/internal/build/retake_test.go b/modules/mesh-delivery-builder/internal/build/retake_test.go index 0999b450..c3b4ba0f 100644 --- a/modules/mesh-delivery-builder/internal/build/retake_test.go +++ b/modules/mesh-delivery-builder/internal/build/retake_test.go @@ -57,7 +57,7 @@ func TestRetakeOnAnEvent(t *testing.T) { Entry: func(context.Context, string) (meshcatalog.CatalogEntry, error) { return meshcatalog.CatalogEntry{Serves: []string{"node-build-go.steps"}}, nil }, - Ask: func(_ context.Context, key string, _ builds.BuilderBuild) error { + Ask: func(_ context.Context, key string, _ builds.BuilderBuild, _ uint64) error { asked = append(asked, key) return nil }} diff --git a/modules/mesh-delivery-builder/internal/build/retakeask.go b/modules/mesh-delivery-builder/internal/build/retakeask.go new file mode 100644 index 00000000..c59f7bab --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/retakeask.go @@ -0,0 +1,162 @@ +package build + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "time" + + occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "mesh-delivery-builder/internal/versions" +) + +// **The operator is asked once a key's event builds are spent** (design 47 G §568, ADR 0673 §3): the seat operator-channel's ask, +// level approve, with the options *Build once more* and *Leave it*, `about` the key so a newer ask replaces it (ADR 0666 §12), +// naming the module, the fingerprint and the last error whole. *Build once more* builds one revision as a person's build +// (BuildAgain, which starts the count again); *Leave it*, a decline or the ask's expiry leaves the key failed. + +// The ids of the ask's options. +const ( + OptionBuildOnceMore = "build-once-more" + OptionLeaveIt = "leave-it" +) + +// AskLasts is how long the ask stays open: the longest an ask that approves may (the seat's own bound). +const AskLasts = occ.ApproveLasts + +// askBuildBound is how long the builder waits for the claim and the asks of the build *Build once more* starts. +const askBuildBound = 15 * time.Second + +// OperatorAsk asks the operator about a key and carries out the answer. +type OperatorAsk struct { + // Client is the builder's asker on the seat operator-channel: its ask published under the builder's own name, its state + // `asked` (the asks it kept and the warrants it acted on), the router's record, the `decided` event. + Client *occ.Client + // Acted keeps which key an ask was made for, beside the revision of the key it was made at (Retaker.Acted). + Acted versions.ReserveStore + Builder *Builder + // Ctx is the builder's own context: the build a person's answer starts lives on it. + Ctx context.Context + Now func() time.Time + Logf func(string, ...any) +} + +func (o *OperatorAsk) logf(f string, a ...any) { + if o.Logf != nil { + o.Logf(f, a...) + } +} + +func (o *OperatorAsk) now() time.Time { + if o.Now != nil { + return o.Now() + } + return time.Now() +} + +// AskID is the id of the ask about one revision of a key: the same revision is the same ask, a newer one a new ask. +func AskID(key string, rev uint64) string { + sum := sha256.Sum256([]byte(fmt.Sprintf("%s@%d", key, rev))) + return "retake-" + hex.EncodeToString(sum[:16]) +} + +// askedOf is the acted key of the key an ask was made for. +func askedOf(id string) string { return versions.ActedKey("retake-ask:" + id) } + +// BuildOnceMoreAct is what *Build once more* binds: the verb and the key's module and fingerprint. +func BuildOnceMoreAct(module, fingerprint string) occ.Act { + return occ.Act{"verb": "build-again", "module": module, "fingerprint": fingerprint} +} + +// Compose is the ask about key as it stands, at revision rev. +func (o *OperatorAsk) Compose(key string, cur builds.BuilderBuild, rev uint64) (oc.Ask, error) { + module, fp, err := builds.ParseBuilderBuildKey(key) + if err != nil { + return oc.Ask{}, err + } + binds, err := occ.ActDigest(BuildOnceMoreAct(module, fp)) + if err != nil { + return oc.Ask{}, err + } + headline := "Build " + module + " once more?" + if r := []rune(headline); len(r) > occ.HeadlineLength { + headline = string(r[:occ.HeadlineLength-1]) + "…" + } + last := []rune(cur.Error) + if len(last) > 120 { + last = append(last[:119], '…') + } + return oc.Ask{ID: AskID(key, rev), Headline: headline, Who: occ.Operator, About: key, + Explanation: fmt.Sprintf("The build of %s failed on a transport fault, and events built it again %d times without it passing.\nLast error: %s", module, cur.EventBuilds, string(last)), + Details: fmt.Sprintf("Key %s\nFingerprint %s\nLast error whole: %s", key, fp, cur.Error), + Expires: occ.FormatTime(o.now().Add(AskLasts)), + OnExpiry: "the key stays failed until builder_build_again or a new fingerprint", + Options: []oc.Option{ + {ID: OptionBuildOnceMore, Label: "Build once more", Does: "the builder builds this fingerprint once more, and counts the build as a person's", Level: occ.Approve, Binds: binds}, + {ID: OptionLeaveIt, Label: "Leave it", Does: "nothing: the key stays failed", Level: occ.Acknowledge}, + }}, nil +} + +// Ask asks the operator about key at revision rev. The same revision asked twice is one ask. +func (o *OperatorAsk) Ask(ctx context.Context, key string, cur builds.BuilderBuild, rev uint64) error { + a, err := o.Compose(key, cur, rev) + if err != nil { + return err + } + k := askedOf(a.ID) + if _, _, found, err := o.Acted.Get(ctx, k); err != nil { + return err + } else if !found { + if _, err := o.Acted.Create(ctx, k, []byte(key)); err != nil && !errors.Is(err, versions.ErrConflict) { + return err + } + } + // An ask published and not marked asked (the write after it failed) is read again: it is one ask. + if e, err := o.Client.State.Get("ask." + a.ID); err != nil { + return err + } else if e != nil { + return nil + } + return o.Client.Ask(a) +} + +// Listen hears the operator's answers: each warrant of this builder's asks is acted on once. +func (o *OperatorAsk) Listen() error { + return o.Client.Listen(o.act, func(h occ.Handled, w oc.Warrant, err error) { + if err != nil { + o.logf("the answer to the ask %s: %s: %v", w.Ask, h, err) + return + } + o.logf("the answer to the ask %s: %s", w.Ask, h) + }) +} + +// act carries out the option the operator chose: Build once more builds the key as a person's build, Leave it nothing. +func (o *OperatorAsk) act(opt oc.Option, w oc.Warrant) error { + if opt.ID != OptionBuildOnceMore { + return nil + } + raw, _, found, err := o.Acted.Get(o.Ctx, askedOf(w.Ask)) + if err != nil { + return err + } + if !found { + return fmt.Errorf("the builder holds no key the ask %s was made for", w.Ask) + } + module, fp, err := builds.ParseBuilderBuildKey(string(raw)) + if err != nil { + return err + } + o.logf("%s: the operator chose %s; built again as a person's build", raw, opt.Label) + go func() { + if res := o.Builder.BuildAgain(o.Ctx, module, fp, "the operator chose "+opt.Label, "operator-channel", askBuildBound); res.Refused != "" { + o.logf("%s: not built again: %s", raw, res.Refused) + } + }() + return nil +} diff --git a/modules/mesh-delivery-builder/internal/build/retakeask_test.go b/modules/mesh-delivery-builder/internal/build/retakeask_test.go new file mode 100644 index 00000000..36a654d8 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/retakeask_test.go @@ -0,0 +1,113 @@ +package build + +import ( + "context" + "encoding/json" + "testing" + "time" + + occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + stdio "git.novox.be/novox/mesh-sdk/go" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" +) + +// askedState is the client's state `asked`. +type askedState struct{ vals map[string][]byte } + +func (s *askedState) Get(key string) (*stdio.StateEntry, error) { + if v, ok := s.vals[key]; ok { + return &stdio.StateEntry{Key: key, Value: v, Revision: 1}, nil + } + return nil, nil +} +func (s *askedState) Create(key string, value any) (uint64, error) { + if _, ok := s.vals[key]; ok { + return 0, stdio.ErrChanged + } + return s.Update(key, value, 0) +} +func (s *askedState) Update(key string, value any, _ uint64) (uint64, error) { + if s.vals == nil { + s.vals = map[string][]byte{} + } + s.vals[key], _ = json.Marshal(value) + return 1, nil +} + +// Design 47 G §568: after three event builds the operator is asked, through the seat operator-channel's ask (level approve, +// *Build once more* and *Leave it*, about the key), and *Build once more* builds the key as a person's build, *Leave it* does not. +func TestTheOperatorIsAskedAndAnswers(t *testing.T) { + for _, option := range []string{OptionLeaveIt, OptionBuildOnceMore} { + t.Run(option, func(t *testing.T) { theOperatorAnswers(t, option) }) + } +} + +func theOperatorAnswers(t *testing.T, option string) { + r := newRig(t, goBundle) + r.consider(t, "modules/m") + seed(t, r.st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.builder", Manifest: mdigest('a'), EventBuilds: 3, + Error: "transport fault: connection, node-build-go ace, dial tcp: connection refused"}) + var published []oc.Ask + acted := &actedStore{} + asker := &OperatorAsk{Builder: r.b, Acted: acted, Ctx: context.Background(), Now: func() time.Time { return time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC) }, + Client: &occ.Client{Module: "mesh-delivery-builder", State: &askedState{}, + Publish: func(subject string, body any, id string) error { + if subject != oc.AcceptSubjectAsk("mesh-delivery-builder") { + t.Errorf("published on %s", subject) + } + published = append(published, body.(oc.Ask)) + return nil + }}} + rt := &Retaker{Builder: r.b, Acted: acted, Ask: asker.Ask, + Entry: nil} + data, _ := json.Marshal(map[string]string{"node": "ace", "why": "returned"}) + for seq := uint64(1); seq <= 2; seq++ { // the same key and revision asked at two events is one ask + if _, err := rt.Heard(context.Background(), "mesh.seat.node-engine.event.manifest-reads-changed.ace", data, seq); err != nil { + t.Fatal(err) + } + r.b.Wait() + } + if len(published) != 1 { + t.Fatalf("asked %d times, want once", len(published)) + } + a := published[0] + if a.About != r.key || len(a.Options) != 2 || a.Options[0].Level != occ.Approve || a.Options[0].Label != "Build once more" || a.Options[1].Label != "Leave it" { + t.Fatalf("the ask = %+v, want about the key with the options Build once more (approve) and Leave it", a) + } + if err := occ.Check(a, time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)); err != nil { + t.Fatalf("the ask does not hold to the seat's checks: %v", err) + } + if err := occ.Glance(a.Explanation); err != nil { + t.Fatal(err) + } + warrant := func(option string) oc.Warrant { + return oc.Warrant{Ask: a.ID, AskDigest: occ.Digest(a), Asker: "mesh-delivery-builder", Outcome: occ.OutcomeChosen, Option: option, Level: levelOf(a, option), + At: occ.FormatTime(time.Date(2026, 11, 11, 12, 5, 0, 0, time.UTC)), By: &oc.Person{Who: occ.Operator, Kind: "telegram", Identity: "op", Verified: "user"}} + } + h, err := asker.Client.Take(warrant(option), asker.act) + if err != nil || h != occ.Acted { + t.Fatalf("%s: %v, %v", option, h, err) + } + for i := 0; i < 50 && option == OptionBuildOnceMore; i++ { // the build the answer starts runs on its own goroutine + r.b.Wait() + if valueOf(t, r.st, r.key).State == builds.BuildDone { + break + } + time.Sleep(time.Millisecond) + } + r.b.Wait() + v := valueOf(t, r.st, r.key) + if option == OptionLeaveIt && (v.State != builds.BuildFailed || v.EventBuilds != 3) { + t.Fatalf("after Leave it the key = %+v, want failed as it was", v) + } + if option == OptionBuildOnceMore && (v.State != builds.BuildDone || v.EventBuilds != 0) { + t.Fatalf("after Build once more the key = %+v, want built as a person's build (event-builds 0)", v) + } +} + +func levelOf(a oc.Ask, option string) string { + o, _ := occ.OptionOf(a, option) + return o.Level +} diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index c25851da..887e62c0 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -113,7 +113,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if m.Module != "mesh-delivery-builder" { t.Errorf("module = %q", m.Module) } - if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory + if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory t.Errorf("state = %+v, want %+v", m.State, want) } // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and @@ -123,8 +123,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 "mesh-delivery-assigner.assignments", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } - if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) { - t.Errorf("uses = %v, want the three build seats", m.Uses) + if want := []string{"node-build-go", "node-build-image", "node-build-node", "operator-channel"}; !reflect.DeepEqual(sorted(m.Uses), want) { + t.Errorf("uses = %v, want the three build seats and operator-channel (design 47 G §568: the ask after three event builds)", m.Uses) } // 1937/AC3: the builder asks the controller whether the artifact store still holds a done key's blobs (held), and no // other tool; it holds no capacity. @@ -144,7 +144,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if want := []string{"mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-forge-bridge.bridge_tags", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests", "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.publish", "seat:container-image-registry.remove", "seat:container-image-registry.versions", "seat:git.tag", - "seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-catalog.entry", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove", + "seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-catalog.entry", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove", "seat:npm-package-registry.versions", "seat:oci-artifact-registry.publish", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } @@ -161,7 +161,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.packages", "node-build-go.toolchain-reported", "node-build-node.toolchain-reported", "node-engine.manifest-reads-changed", "node-engine.running", "npm-package-registry.kept", "npm-package-registry.published", "npm-package-registry.refused", "npm-package-registry.removed", - "oci-artifact-registry.kept", "oci-artifact-registry.published", "oci-artifact-registry.refused", "oci-artifact-registry.removed"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + "oci-artifact-registry.kept", "oci-artifact-registry.published", "oci-artifact-registry.refused", "oci-artifact-registry.removed", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 9577bc12..069a757d 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -60,6 +60,7 @@ "node-build-node.toolchain-reported", "node-engine.manifest-reads-changed", "node-engine.running", + "operator-channel.decided", "npm-package-registry.kept", "npm-package-registry.published", "npm-package-registry.refused", @@ -86,6 +87,7 @@ "seat:mesh-catalog.entry", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", + "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", @@ -120,6 +122,7 @@ { "name": "reserved", "history": 1 }, { "name": "publish-queue", "history": 1, "max-mib": 4 }, { "name": "acted", "history": 1, "max-mib": 1 }, + { "name": "asked", "history": 1 }, { "name": "reasks", "history": 1 }, { "name": "versions", "history": 1 }, { "name": "first-views", "history": 1 } @@ -159,7 +162,8 @@ "uses": [ "node-build-go", "node-build-image", - "node-build-node" + "node-build-node", + "operator-channel" ], "own-secrets": { "broker": "${dir:state}/broker" From 12e3f2dc483ea260d7067dc2d9369a6a546f15a3 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:28:08 +0100 Subject: [PATCH 09/10] BL17d: builder_build answers a module's build keys whole; builder_builds_since takes module; builder_stuck's fingerprint pinned whole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit builder_build reads the builds and asked states, and the tools bundle follows the catalog half itself (registrations for the watched branches, revision per branch) for `newest`; the manifest names the tool and the two seat verbs. Builds: design 47 G §589, A-87 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-builder-tools/build.go | 79 +++++++ .../mesh-delivery-builder-tools/build_test.go | 32 +++ .../cmd/mesh-delivery-builder-tools/main.go | 20 +- .../internal/build/buildread.go | 222 ++++++++++++++++++ .../internal/build/buildread_test.go | 83 +++++++ .../internal/build/run_test.go | 11 +- .../internal/build/since.go | 8 +- .../internal/build/stuck.go | 49 ++-- .../mesh-delivery-builder/manifest_test.go | 4 +- modules/mesh-delivery-builder/module.json | 3 + 10 files changed, 482 insertions(+), 29 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build_test.go create mode 100644 modules/mesh-delivery-builder/internal/build/buildread.go create mode 100644 modules/mesh-delivery-builder/internal/build/buildread_test.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build.go new file mode 100644 index 00000000..a91ec8a2 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build.go @@ -0,0 +1,79 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscalls" + + "mesh-delivery-builder/internal/build" +) + +// catalogWithin is how long a call waits for the catalog half's first view of one branch. +var catalogWithin = 30 * time.Second + +// buildTool is builder_build (design 47 G §589): a module's build keys whole. +func buildTool(r *build.BuildReader) stdio.Tool { + return stdio.Tool{Name: build.BuildTool, + Description: "A module's build keys, whole, newest first (at most 20): per key its fingerprint (whole, as the build-input fingerprint the catalog graph " + + "entry carries it), state (queued, building, done or failed), revision and since when, the step and its node, the error whole, event-builds " + + "(the revisions events started since the last a person started; after three the operator is asked) and the open ask; and newest, the watched " + + "branches whose newest catalog graph entry of the module names that fingerprint, read from the catalog half. A module none of whose keys the newest " + + "entry names says so, naming the entry's fingerprint. With fingerprint, that key alone. Which node wants which fingerprint is the assigner's " + + "to answer (the seat mesh-wants). Read-only.", + Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module"}, + "fingerprint": map[string]any{"type": "string", "description": "one key: its fingerprint, src2:"}}}, + Run: func(args map[string]any) (any, error) { + module, _ := args["module"].(string) + fp, _ := args["fingerprint"].(string) + return r.Build(module, fp) + }} +} + +// newestVia reads the fingerprint of a module's newest catalog graph entry on each of its watched branches, from the catalog +// half through the seat mesh-catalog (the verb registrations for the branches, the verb revision for each branch's view), as +// the delivery view's tools read it: once per call (design 47 G §532, §589, A-87). +func newestVia(ask meshcatalogclient.Call) func(module string) (map[string]string, error) { + return func(module string) (map[string]string, error) { + raw, err := ask(meshcatalogclient.RegistrationsVerb, meshcatalog.RegistrationsInput{Module: module}) + if err != nil { + return nil, fmt.Errorf("the registrations of %s: %w", module, err) + } + answer, said, ok := buscalls.Unwrap(raw) + if !ok { + return nil, fmt.Errorf("%s answered: %s", meshcatalogclient.RegistrationsVerb, said) + } + var regs meshcatalog.RegistrationsAnswer + if err := json.Unmarshal(answer, ®s); err != nil { + return nil, fmt.Errorf("%s answered no registrations: %w", meshcatalogclient.RegistrationsVerb, err) + } + out := map[string]string{} + for _, reg := range regs.Registrations { + if reg.Module != module { + continue + } + for _, branch := range reg.Branches { + c, err := meshcatalogclient.NewCatalog(ask, branch) + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), catalogWithin) + err = c.Start(ctx) + cancel() + if err != nil { + return nil, fmt.Errorf("the catalog half of %s has no first view: %w", branch, err) + } + if e, ok := c.Entry(module); ok && e.Fp != "" { + out[branch] = e.Fp + } + } + } + return out, nil + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build_test.go new file mode 100644 index 00000000..f18c8ad0 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// Design 47 G §589, A-87: the tools bundle reads each watched branch's newest catalog graph entry of the module from the catalog +// half itself: the verb registrations for the branches, the verb revision for each branch's view. +func TestNewestIsReadFromTheCatalogHalfOfEachWatchedBranch(t *testing.T) { + fp := map[string]string{"main": "src2:" + strings.Repeat("1", 64), "feature": "src2:" + strings.Repeat("2", 64)} + ask := func(key string, body any) (json.RawMessage, error) { + switch key { + case meshcatalogclient.RegistrationsVerb: + return json.Marshal(meshcatalog.RegistrationsAnswer{Registrations: []meshcatalog.Registration{{Module: "m", Branches: []string{"main", "feature"}}}}) + case meshcatalogclient.RevisionVerb: + in := body.(meshcatalog.RevisionInput) + return json.Marshal(meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: 3, Kind: meshcatalogclient.KindTrunk}, + Entries: []meshcatalog.CatalogEntry{{Module: "m", Fp: fp[in.Branch]}, {Module: "z", Fp: "src2:" + strings.Repeat("9", 64)}}}) + } + t.Fatalf("asked %s", key) + return nil, nil + } + got, err := newestVia(ask)("m") + if err != nil || len(got) != 2 || got["main"] != fp["main"] || got["feature"] != fp["feature"] { + t.Fatalf("newest = %v, %v, want each branch's entry fingerprint whole", got, err) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index cc4309b2..b42c7e53 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -156,15 +156,16 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q "fingerprint, source revision, repository, directory, commit and ref, the build node, when it ended, its outcome (done, failed or " + "queued), the manifest built, the bases it was built against and each artifact made with its digest and reference; a failed " + "build carries its error and the step it was at. more says the " + - "log held more than one answer carries: ask again from the last build's time. What the operator's command nox builds reads. Read-only.", + "log held more than one answer carries: ask again from the last build's time. With module, that module's builds only. What the operator's command nox builds reads. Read-only.", Input: map[string]any{"type": "object", "required": []string{"since"}, "properties": map[string]any{ - "since": map[string]any{"type": "string", "description": "RFC 3339 time; the builds that ended at or after it"}}}, + "since": map[string]any{"type": "string", "description": "RFC 3339 time; the builds that ended at or after it"}, + "module": map[string]any{"type": "string", "description": "only this module's builds"}}}, Run: func(args map[string]any) (any, error) { at, err := time.Parse(time.RFC3339Nano, str(args, "since")) if err != nil { return nil, fmt.Errorf("since %q is not an RFC 3339 time: %w", str(args, "since"), err) } - return build.BuildsSince(since, at) + return build.BuildsSince(since, at, str(args, "module")) }}, {Name: build.AnnounceAgainTool, Description: "Announce a done build again (take in again, compose again): the builder writes the done key of module and fingerprint " + @@ -363,6 +364,8 @@ func main() { return e.Value, true, nil }} tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr) + tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds")), Asked: read(stdio.State("asked")), + Newest: newestVia(stdio.Ask), Steps: r})) tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) qs, as := stdio.State(versions.PublishQueueState), stdio.State(versions.ActedState) tools = append(tools, republishTools(readerOf(qs), readerOf(as), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) @@ -394,3 +397,14 @@ func main() { os.Exit(1) } } + +// entryOf reads a key with its revision and when it was written. +func entryOf(s stdio.KeptState) func(key string) (json.RawMessage, uint64, time.Time, error) { + return func(key string) (json.RawMessage, uint64, time.Time, error) { + e, err := s.Get(key) + if err != nil || e == nil { + return nil, 0, time.Time{}, err + } + return e.Value, e.Revision, written(s, key, e.Revision), nil + } +} diff --git a/modules/mesh-delivery-builder/internal/build/buildread.go b/modules/mesh-delivery-builder/internal/build/buildread.go new file mode 100644 index 00000000..1d9ec73a --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/buildread.go @@ -0,0 +1,222 @@ +package build + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "time" + + occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" +) + +// **builder_build answers a module's build keys whole** (design 47 G §589, A-87): with `fingerprint` that key whole, with +// `module` alone the module's keys newest first, at most MostBuilds, each with its fingerprint whole, its state, revision and +// since when, its step and node, its error whole, event-builds and the open ask (§568), and `newest` naming each watched branch +// whose newest catalog graph entry of the module names that fingerprint, read from the catalog half by the tools bundle. + +// BuildTool is the builder's read tool of a module's build keys. +const BuildTool = "builder_build" + +// MostBuilds is how many keys of a module builder_build answers by module alone. +const MostBuilds = 20 + +// BuildRow is one build key as builder_build answers it. +type BuildRow struct { + Key string `json:"key"` + Fingerprint string `json:"fingerprint"` + State string `json:"state"` + Revision uint64 `json:"revision,omitempty"` + Since time.Time `json:"since,omitzero"` + Step string `json:"step,omitempty"` + Node string `json:"node,omitempty"` + Error string `json:"error,omitempty"` + EventBuilds int `json:"event-builds"` + // Ask is the open ask of the operator about the key (§568). + Ask *OpenAsk `json:"ask,omitempty"` + // Newest names each watched branch whose newest catalog graph entry of the module names this fingerprint. + Newest []string `json:"newest,omitempty"` +} + +// OpenAsk is the ask the operator has not answered. +type OpenAsk struct { + ID string `json:"id"` + Asked string `json:"asked"` + Expires string `json:"expires"` +} + +// BuildAnswer is what builder_build answers. +type BuildAnswer struct { + Module string `json:"module"` + Builds []BuildRow `json:"builds"` + // Newest is the fingerprint of the module's newest catalog graph entry on each watched branch. + Newest map[string]string `json:"newest-entries,omitempty"` + // Note says what no key shows: that none of the keys is the newest entry's, naming its fingerprint. + Note string `json:"note,omitempty"` +} + +// BuildReader reads what builder_build answers. +type BuildReader struct { + Builds StateRead + // Entry is a key's value with its revision and when it was written; nil leaves both out. + Entry func(key string) (json.RawMessage, uint64, time.Time, error) + // Asked is the builder's state asked, which holds the asks the operator was asked. + Asked StateRead + // Newest is the fingerprint of the module's newest catalog graph entry on each watched branch, read from the catalog half. + Newest func(module string) (map[string]string, error) + Steps StuckReader + index map[string]map[string]string +} + +// Build answers for module, and for one fingerprint when fingerprint is not empty. +func (r *BuildReader) Build(module, fingerprint string) (BuildAnswer, error) { + out := BuildAnswer{Module: module, Builds: []BuildRow{}} + if module == "" { + return out, errors.New("name the module (module)") + } + all, err := r.Builds.Keys() + if err != nil { + return out, err + } + var rows []BuildRow + for _, key := range all { + mod, fp, err := builds.ParseBuilderBuildKey(key) + if err != nil || mod != module || (fingerprint != "" && fp != fingerprint) { + continue + } + raw, rev, at, err := r.read(key) + if err != nil { + return out, fmt.Errorf("%s cannot be read: %w", key, err) + } + if raw == nil { + continue + } + v, err := builds.DecodeBuilderBuild(raw) + if err != nil { + return out, fmt.Errorf("%s holds what is not a build: %w", key, err) + } + row := BuildRow{Key: key, Fingerprint: fp, State: v.State, Revision: rev, Since: at, Step: v.Step, Error: v.Error, EventBuilds: v.EventBuilds} + if _, _, node, _, ok := builds.TransportFaultOf(v.Error); ok && node != "no node" { + row.Node = node + } + if row.Node == "" && v.State == builds.BuildBuilding { + row.Node = r.stepNode(v) + } + rows = append(rows, row) + } + if fingerprint != "" && len(rows) == 0 { + return out, fmt.Errorf("the builder holds no build key of %s at %s", module, fingerprint) + } + sort.Slice(rows, func(i, j int) bool { + if !rows[i].Since.Equal(rows[j].Since) { + return rows[i].Since.After(rows[j].Since) + } + return rows[i].Key > rows[j].Key + }) + if len(rows) > MostBuilds { + rows = rows[:MostBuilds] + } + asks := r.openAsks() + for i := range rows { + rows[i].Ask = asks[rows[i].Key] + } + if r.Newest != nil { + newest, err := r.Newest(module) + if err != nil { + return out, fmt.Errorf("the catalog half's newest entry of %s cannot be read: %w", module, err) + } + out.Newest = newest + named := false + for i := range rows { + for branch, fp := range newest { + if fp == rows[i].Fingerprint { + rows[i].Newest, named = append(rows[i].Newest, branch), true + } + } + slices.Sort(rows[i].Newest) + } + if !named && len(newest) > 0 { + var says []string + for branch, fp := range newest { + says = append(says, branch+" names "+fp) + } + slices.Sort(says) + out.Note = "no build key of " + module + " is the newest entry's: " + strings.Join(says, "; ") + } + } + out.Builds = append(out.Builds, rows...) + return out, nil +} + +// stepNode is the node a building key's step runs on, read from its asks' steps in the build modules' steps; empty when none runs. +func (r *BuildReader) stepNode(v builds.BuilderBuild) string { + if len(v.Asks) == 0 || len(r.Steps.Steps) == 0 { + return "" + } + if r.index == nil { + idx, err := r.Steps.stepIndex() + if err != nil { + return "" + } + r.index = idx + } + languages := make([]string, 0, len(v.Asks)) + for l := range v.Asks { + languages = append(languages, l) + } + slices.Sort(languages) + for _, l := range languages { + bm, _ := buildstep.BuildModuleOf(l) + st, ok := r.Steps.Steps[bm] + k, found := r.index[bm][v.Asks[l]] + if !ok || !found { + continue + } + if raw, _, err := st.Get(k); err == nil && raw != nil { + if step, err := buildstep.DecodeStep(raw); err == nil && step.Node != "" { + return step.Node + } + } + } + return "" +} + +func (r *BuildReader) read(key string) (json.RawMessage, uint64, time.Time, error) { + if r.Entry != nil { + return r.Entry(key) + } + raw, at, err := r.Builds.Get(key) + return raw, 0, at, err +} + +// openAsks are the asks kept in the state asked that nobody answered yet, by the key each is about. +func (r *BuildReader) openAsks() map[string]*OpenAsk { + out := map[string]*OpenAsk{} + if r.Asked.Keys == nil { + return out + } + keys, err := r.Asked.Keys() + if err != nil { + return out + } + for _, k := range keys { + raw, _, err := r.Asked.Get(k) + if err != nil || raw == nil { + continue + } + var kept occ.Kept + if json.Unmarshal(raw, &kept) != nil || kept.Ended != "" || kept.Acted || kept.Ask.About == "" { + continue + } + if t, err := occ.Time(kept.Ask.Expires); err == nil && !t.IsZero() && time.Now().After(t) { + continue + } + out[kept.Ask.About] = &OpenAsk{ID: kept.Ask.ID, Asked: kept.Asked.UTC().Format(time.RFC3339), Expires: kept.Ask.Expires} + } + return out +} diff --git a/modules/mesh-delivery-builder/internal/build/buildread_test.go b/modules/mesh-delivery-builder/internal/build/buildread_test.go new file mode 100644 index 00000000..92be0c20 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/buildread_test.go @@ -0,0 +1,83 @@ +package build + +import ( + "encoding/json" + "sort" + "strings" + "testing" + "time" + + occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" +) + +const fp2 = "src2:2222222222222222222222222222222222222222222222222222222222222222" + +func stateOf(vals map[string][]byte) StateRead { + return StateRead{ + Keys: func() ([]string, error) { + var ks []string + for k := range vals { + ks = append(ks, k) + } + sort.Strings(ks) + return ks, nil + }, + Get: func(k string) (json.RawMessage, time.Time, error) { return vals[k], time.Time{}, nil }} +} + +// Design 47 G §589: builder_build by module answers its keys newest first with whole fingerprints, event-builds and the open +// ask, and names the watched branches whose newest entry names a key's fingerprint; a module none of whose keys the newest +// entry names says so; with a fingerprint, that key alone. +func TestBuilderBuildAnswersAModulesKeysWhole(t *testing.T) { + failed, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildFailed, EventBuilds: 3, Manifest: mdigest('a'), + Error: "transport fault: connection, node-build-go ace, dial tcp: connection refused"}) + done, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest('a'), Artifacts: []builds.BuiltArtifact{{Name: "x", Kind: "file", Digest: "sha256:" + strings.Repeat("a", 64)}}}) + k1, _ := builds.BuilderBuildKey("m", fp1) + k2, _ := builds.BuilderBuildKey("m", fp2) + other, _ := builds.BuilderBuildKey("other", fp1) + vals := map[string][]byte{k1: failed, k2: done, other: done} + ask := occ.Kept{Ask: oc.Ask{ID: "retake-1", About: k1, Expires: occ.FormatTime(time.Now().Add(time.Hour))}, Asked: time.Now()} + askRaw, _ := json.Marshal(ask) + r := &BuildReader{Builds: stateOf(vals), Asked: stateOf(map[string][]byte{"ask.retake-1": askRaw}), + Newest: func(string) (map[string]string, error) { return map[string]string{"main": fp1, "feature": fp2}, nil }} + a, err := r.Build("m", "") + if err != nil || len(a.Builds) != 2 { + t.Fatalf("builds = %+v, %v, want the module's two keys", a, err) + } + var row BuildRow + for _, b := range a.Builds { + if b.Fingerprint == fp1 { + row = b + } + } + if row.Fingerprint != fp1 || len(row.Fingerprint) != len("src2:")+64 || row.State != builds.BuildFailed || row.EventBuilds != 3 || row.Node != "ace" || + !strings.HasPrefix(row.Error, "transport fault: connection, node-build-go ace, dial tcp") || row.Ask == nil || row.Ask.ID != "retake-1" || len(row.Newest) != 1 || row.Newest[0] != "main" { + t.Errorf("the failed key = %+v, want its fingerprint whole, its error whole, event-builds 3, the open ask and newest main", row) + } + if one, err := r.Build("m", fp2); err != nil || len(one.Builds) != 1 || one.Builds[0].Fingerprint != fp2 || one.Builds[0].Newest[0] != "feature" { + t.Errorf("one key = %+v, %v", one, err) + } + r.Newest = func(string) (map[string]string, error) { + return map[string]string{"main": "src2:" + strings.Repeat("c", 64)}, nil + } + if n, err := r.Build("m", ""); err != nil || !strings.Contains(n.Note, "no build key of m is the newest entry's") || !strings.Contains(n.Note, "main names src2:"+strings.Repeat("c", 64)) { + t.Errorf("a module none of whose keys the newest entry names: %+v, %v", n, err) + } +} + +// Design 47 G §589: builder_stuck answers each key's fingerprint whole, as the build-input fingerprint the catalog graph entry carries it. +func TestBuilderStuckAnswersTheFingerprintWhole(t *testing.T) { + building, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildBuilding, Step: "asked", Manifest: mdigest('a')}) + k, _ := builds.BuilderBuildKey("m", fp1) + out, err := StuckReader{Builds: stateOf(map[string][]byte{k: building})}.Stuck("m") + if err != nil { + t.Fatal(err) + } + rows := out["stuck"].([]Stuck) + if len(rows) != 1 || rows[0].FP != fp1 { + t.Fatalf("stuck = %+v, want the fingerprint %s whole", rows, fp1) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index ce3d66f3..3955c800 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -270,12 +270,19 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7 page.Entries = append(page.Entries, stdio.LogEntry{Sequence: uint64(i + 1), Key: e.Module, Value: raw, At: e.At}) } page.Next = 3 - since, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At) + since, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At, "") if err != nil || len(since.Builds) != 1 || since.Builds[0].Event != builds.BuildEndedEvent || len(since.Builds[0].Made) != 3 || !since.More { t.Errorf("builds since = %+v, %v, want the one end and more", since, err) } + // With module (design 47 G §589) the builds of that module alone. + if other, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At, "another"); err != nil || len(other.Builds) != 0 { + t.Errorf("builds since of another module = %+v, %v, want none", other, err) + } + if mine, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At, logged[0].Module); err != nil || len(mine.Builds) != 1 { + t.Errorf("builds since of the module = %+v, %v, want its one end", mine, err) + } page.Entries[0].Value = []byte(`{"v":0}`) - if _, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At); err == nil { + if _, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At, ""); err == nil { t.Error("an entry that does not read was skipped, want it refused") } if v.State != builds.BuildDone || v.By != "novox.builder" || v.Manifest != manifestD || len(v.Artifacts) != 3 { diff --git a/modules/mesh-delivery-builder/internal/build/since.go b/modules/mesh-delivery-builder/internal/build/since.go index 8875b37d..d459a9b8 100644 --- a/modules/mesh-delivery-builder/internal/build/since.go +++ b/modules/mesh-delivery-builder/internal/build/since.go @@ -2,6 +2,7 @@ package build import ( "fmt" + "slices" "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" @@ -16,9 +17,9 @@ type LogSince func(at time.Time, keys string, limit int) (*stdio.LogPage, error) // BuildsSince answers builds.BuildsSinceTool (ADR 0571 clause 9): every ended build in the builder's log from since on, // oldest first, read through the runtime's log verbs; More when the page was not the log's last, so the caller asks again -// from the last build's time. An entry the log holds that does not read is refused, naming its sequence: a reader that +// from the last build's time. With module (design 47 G §589) only that module's builds are answered. An entry the log holds that does not read is refused, naming its sequence: a reader that // skipped it would register too little. -func BuildsSince(read LogSince, since time.Time) (builds.BuildsSince, error) { +func BuildsSince(read LogSince, since time.Time, module string) (builds.BuildsSince, error) { out := builds.BuildsSince{Since: since.UTC(), Builds: []builds.BuildLogEntry{}} page, err := read(since, "", BuildsSinceLimit) if err != nil { @@ -38,6 +39,9 @@ func BuildsSince(read LogSince, since time.Time) (builds.BuildsSince, error) { if ended := builds.EndedSince(entries, since); ended != nil { out.Builds = ended } + if module != "" { + out.Builds = slices.DeleteFunc(out.Builds, func(e builds.BuildLogEntry) bool { return e.Module != module }) + } out.More = page.Next != 0 return out, nil } diff --git a/modules/mesh-delivery-builder/internal/build/stuck.go b/modules/mesh-delivery-builder/internal/build/stuck.go index eafe7379..5a6666b9 100644 --- a/modules/mesh-delivery-builder/internal/build/stuck.go +++ b/modules/mesh-delivery-builder/internal/build/stuck.go @@ -313,26 +313,9 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) { if r.Now != nil { now = r.Now } - // Steps are found by the ask's id, which their key ends in: one index per build module, no value read until an ask names it. - stepKeys := map[string]map[string]string{} - for bm, s := range r.Steps { - ks, err := s.Keys() - if err != nil { - return nil, fmt.Errorf("the steps of %s: %w", bm, err) - } - byID := make(map[string]string, len(ks)) - for _, k := range ks { - _, _, id, err := buildstep.ParseStepKey(k) - if err != nil { - // A key neither form parses is named, never passed over in silence (design 47 G §484). - if r.Logf != nil { - r.Logf("builder_stuck: the key %q of the steps of %s is a step key of neither form, .. or .: %v", k, bm, err) - } - continue - } - byID[id] = k - } - stepKeys[bm] = byID + stepKeys, err := r.stepIndex() + if err != nil { + return nil, err } // Narrowed by the key (the module) before any value is read; the state is in the value, so every matching key is read, in // parallel, at most MaxReads of them. @@ -429,6 +412,32 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) { return answer, nil } +// stepIndex finds the build modules' steps by the ask's id, which their key ends in: one index per build module, no value read +// until an ask names it. +func (r StuckReader) stepIndex() (map[string]map[string]string, error) { + stepKeys := map[string]map[string]string{} + for bm, s := range r.Steps { + ks, err := s.Keys() + if err != nil { + return nil, fmt.Errorf("the steps of %s: %w", bm, err) + } + byID := make(map[string]string, len(ks)) + for _, k := range ks { + _, _, id, err := buildstep.ParseStepKey(k) + if err != nil { + // A key neither form parses is named, never passed over in silence (design 47 G §484). + if r.Logf != nil { + r.Logf("builder_stuck: the key %q of the steps of %s is a step key of neither form, .. or .: %v", k, bm, err) + } + continue + } + byID[id] = k + } + stepKeys[bm] = byID + } + return stepKeys, nil +} + // StuckBuildModules are the build modules whose steps builder_stuck reads. var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule} diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 887e62c0..533f28e6 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -144,7 +144,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if want := []string{"mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-forge-bridge.bridge_tags", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests", "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.publish", "seat:container-image-registry.remove", "seat:container-image-registry.versions", "seat:git.tag", - "seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-catalog.entry", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove", + "seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove", "seat:npm-package-registry.versions", "seat:oci-artifact-registry.publish", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } @@ -171,7 +171,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 t.Errorf("secrets-owner = %q", m.SecretsOwner) } // 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again. - if want := []string{"builder_announce_again", "builder_ask_again", "builder_bounds", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views", + if want := []string{"builder_announce_again", "builder_ask_again", "builder_bounds", "builder_build", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views", "builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserved", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 069a757d..6994fab0 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -15,6 +15,7 @@ "builder_announce_again", "builder_ask_again", "builder_bounds", + "builder_build", "builder_build_again", "builder_builds_since", "builder_engine_reads", @@ -85,6 +86,8 @@ "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-catalog.entry", + "seat:mesh-catalog.registrations", + "seat:mesh-catalog.revision", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", From dc024ae5527d41c04ab721dbc43eaf1242787c73 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:33:20 +0100 Subject: [PATCH 10/10] BL17e: the read of an ask's steps is a kept call: a fault ends no key, the watch is taken up again from its last revision, past five minutes unanswered it is call-unanswered on a bound key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JetStreamAsks.WatchAsk opens the watch as a kept call and takes a closed watch up after the last revision heard; the first fault arms a key of the state bounds with its own time to live (StepsBoundKey), whose end while the read is unanswered raises call-unanswered about the ask, and the answer ends it. Builds: design 47 G §568, A-77 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-delivery-builder/buildrun/asks.go | 108 ++++++++++++++---- .../buildrun/steps_resume_test.go | 88 ++++++++++++++ .../cmd/mesh-delivery-builder/main.go | 10 +- .../cmd/mesh-delivery-builder/stepsread.go | 95 +++++++++++++++ .../mesh-delivery-builder/stepsread_test.go | 43 +++++++ .../internal/versions/publish.go | 16 +++ 6 files changed, 336 insertions(+), 24 deletions(-) create mode 100644 modules/mesh-delivery-builder/buildrun/steps_resume_test.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepsread.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepsread_test.go diff --git a/modules/mesh-delivery-builder/buildrun/asks.go b/modules/mesh-delivery-builder/buildrun/asks.go index 7db40ad7..85efdecf 100644 --- a/modules/mesh-delivery-builder/buildrun/asks.go +++ b/modules/mesh-delivery-builder/buildrun/asks.go @@ -299,6 +299,14 @@ func (k Asker) Passed(step buildstep.Step, err error) (buildstep.Step, error) { // of `*.` in the build module's steps bucket. type JetStreamAsks struct { JS jetstream.JetStream + // Unanswered is told each time the read of an ask's steps meets a transport fault it keeps waiting out (design 47 G §568, ADR + // 0668 §3): the builder arms the call's bound, a key with its own time to live (A-77), and raises call-unanswered when it + // ends unanswered. Answered is told when the read is answered again. Both may be nil. + Unanswered func(module, askID string, fault error) + Answered func(module, askID string) + // OpenWatch opens the watch of keys filtered by pattern in the build module's steps state, from the revision after `after` + // when after is not 0; nil is the bus's. + OpenWatch func(ctx context.Context, module, pattern string, after uint64) (jetstream.KeyWatcher, error) } // PublishAsk is AskPublisher's. @@ -309,46 +317,100 @@ func (s JetStreamAsks) PublishAsk(ctx context.Context, subject, id string, data return err } -// WatchAsk is AskWaiter's: the watch hands over what the keys hold now, so a step that ended before it began is read at -// once, and every change after. -func (s JetStreamAsks) WatchAsk(ctx context.Context, module, askID string) (<-chan buildstep.Step, error) { - pattern, err := buildstep.AskStepWatch(askID) - if err != nil { - return nil, err +// openWatch opens the watch of pattern in the steps state of module, from the revision after `after` when it is not 0. +func (s JetStreamAsks) openWatch(ctx context.Context, module, pattern string, after uint64) (jetstream.KeyWatcher, error) { + if s.OpenWatch != nil { + return s.OpenWatch(ctx, module, pattern, after) } kv, err := s.JS.KeyValue(ctx, StepsBucket(module)) if err != nil { return nil, fmt.Errorf("the bucket %s: %w", StepsBucket(module), err) } - w, err := kv.Watch(ctx, pattern) + if after > 0 { + return kv.Watch(ctx, pattern, jetstream.ResumeFromRevision(after)) + } + return kv.Watch(ctx, pattern) +} + +// transportFault says err is a transport fault of ADR 0664 §1's classes or the read's own deadline: nothing was answered. +func transportFault(err error) bool { + return busretry.Undelivered(err) || busretry.TimedOut(err) +} + +// openKept is a kept call: the watch is opened, and opened again while the bus answers nothing, until it is or ctx ends +// (design 47 G §568: a transport fault of the builder's own read ends no build key). Each fault past the call's own asking again +// is told to Unanswered; the answer, to Answered. An error that is no transport fault is the caller's. +func (s JetStreamAsks) openKept(ctx context.Context, module, askID, pattern string, after uint64) (jetstream.KeyWatcher, error) { + for { + w, err := busretry.DoRepeatable("opening the watch of the ask "+askID+"'s step on "+module, func() (jetstream.KeyWatcher, error) { + return s.openWatch(ctx, module, pattern, after) + }, nil) + if err == nil { + if s.Answered != nil { + s.Answered(module, askID) + } + return w, nil + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + if !transportFault(err) { + return nil, err + } + if s.Unanswered != nil { + s.Unanswered(module, askID, err) + } + } +} + +// WatchAsk is AskWaiter's: the watch hands over what the keys hold now, so a step that ended before it began is read at +// once, and every change after. A watch that cannot be opened for a transport fault is opened again once the bus answers, and one +// that closes is taken up again from the revision after the last it handed over (design 47 G §568), so a fault fails no key. +func (s JetStreamAsks) WatchAsk(ctx context.Context, module, askID string) (<-chan buildstep.Step, error) { + pattern, err := buildstep.AskStepWatch(askID) + if err != nil { + return nil, err + } + w, err := s.openKept(ctx, module, askID, pattern, 0) if err != nil { return nil, err } out := make(chan buildstep.Step) go func() { defer close(out) - defer w.Stop() + var last uint64 for { - select { - case <-ctx.Done(): - return - case e, ok := <-w.Updates(): - if !ok { - return - } - if e == nil || e.Operation() != jetstream.KeyValuePut { - continue - } - step, err := buildstep.DecodeStep(e.Value()) - if err != nil { - continue // not a step (a node's pause key never matches an ask id; anything else is not this ask's) - } + closed := false + for !closed { select { - case out <- step: case <-ctx.Done(): + _ = w.Stop() return + case e, ok := <-w.Updates(): + if !ok { + closed = true + break + } + if e == nil || e.Operation() != jetstream.KeyValuePut { + continue + } + last = max(last, e.Revision()) + step, err := buildstep.DecodeStep(e.Value()) + if err != nil { + continue // not a step (a node's pause key never matches an ask id; anything else is not this ask's) + } + select { + case out <- step: + case <-ctx.Done(): + _ = w.Stop() + return + } } } + _ = w.Stop() + if w, err = s.openKept(ctx, module, askID, pattern, last+1); err != nil { + return + } } }() return out, nil diff --git a/modules/mesh-delivery-builder/buildrun/steps_resume_test.go b/modules/mesh-delivery-builder/buildrun/steps_resume_test.go new file mode 100644 index 00000000..cb8c4203 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/steps_resume_test.go @@ -0,0 +1,88 @@ +package buildrun + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/busretry" +) + +type fakeEntry struct { + jetstream.KeyValueEntry + rev uint64 + value []byte +} + +func (e fakeEntry) Revision() uint64 { return e.rev } +func (e fakeEntry) Value() []byte { return e.value } +func (e fakeEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut } + +type fakeWatcher struct { + jetstream.KeyWatcher + updates chan jetstream.KeyValueEntry +} + +func (w fakeWatcher) Updates() <-chan jetstream.KeyValueEntry { return w.updates } +func (w fakeWatcher) Stop() error { return nil } + +// Design 47 G §568: the read of an ask's steps is a kept call. A transport fault opening the watch ends no key (the wait is +// not failed): it is opened again once the bus answers, said unanswered meanwhile and answered after; and a watch that closes +// is taken up again from the revision after the last it handed over, so the step that ended meanwhile is heard. +func TestTheWatchOfAStepResumesAfterAFault(t *testing.T) { + t.Cleanup(busretry.Waits(20*time.Millisecond, time.Millisecond, 2*time.Millisecond)) + passed, _ := buildstep.EncodeStep(buildstep.Step{ID: "mvl0frv18d1958", Tool: "build", Node: "ace", State: buildstep.StepPassed}) + running, _ := buildstep.EncodeStep(buildstep.Step{ID: "mvl0frv18d1958", Tool: "build", Node: "ace", State: buildstep.StepRunning}) + var mu sync.Mutex + var opens []uint64 + var unanswered, answered int + asks := JetStreamAsks{ + Unanswered: func(string, string, error) { mu.Lock(); unanswered++; mu.Unlock() }, + Answered: func(string, string) { mu.Lock(); answered++; mu.Unlock() }, + OpenWatch: func(_ context.Context, _, _ string, after uint64) (jetstream.KeyWatcher, error) { + mu.Lock() + defer mu.Unlock() + opens = append(opens, after) + ch := make(chan jetstream.KeyValueEntry, 2) + // The bus answers nothing past the kept call's own asking again (its bound), so the fault is told and the watch tried again. + switch n := len(opens); { + case n <= 40: + return nil, nats.ErrNoResponders + case n == 41: + ch <- fakeEntry{rev: 5, value: running} + close(ch) // the watch closes under the wait + default: + ch <- fakeEntry{rev: 6, value: passed} + } + return fakeWatcher{updates: ch}, nil + }} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + steps, err := asks.WatchAsk(ctx, "mesh-build-go", "mvl0frv18d1958") + if err != nil { + t.Fatalf("a transport fault opening the watch failed the wait: %v", err) + } + var states []string + for s := range steps { + states = append(states, s.State) + if buildstep.Ended(s) { + break + } + } + mu.Lock() + defer mu.Unlock() + if len(states) != 2 || states[0] != buildstep.StepRunning || states[1] != buildstep.StepPassed { + t.Fatalf("steps heard = %v, want running then passed", states) + } + if len(opens) != 42 || opens[41] != 6 { + t.Errorf("opens after revisions %v, want the last taken up again after the last revision heard (6)", opens) + } + if unanswered == 0 || answered != 2 { + t.Errorf("unanswered %d, answered %d, want the fault said and each answer said", unanswered, answered) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 0b856d1a..42179b45 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -140,7 +140,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro tools := &buildrun.ToolchainSet{} // Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot // (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity. - asks := buildrun.JetStreamAsks{JS: js} + // The read of an ask's steps is a kept call (design 47 G §568): a fault ends no key, and past five minutes unanswered it is a + // finding; its bound is a key of the state bounds, bound once the bus holds it (set below, with the findings). + stepReads := &stepsRead{bounds: func() jetstream.KeyValue { return nil }, bound: meshconditionsclient.CallUnansweredAfter, now: time.Now, logf: logger.Printf} + asks := buildrun.JetStreamAsks{JS: js, Unanswered: stepReads.unanswered, Answered: stepReads.answered} // The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the wait for its step at once, // and the lane asks its part again (Builder.waitOnce) instead of waiting for the check bound. // Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the @@ -181,6 +184,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // A publish that ends badly is a finding (design 51 *Health and conditions*): a holder's refused, a version no holder // answered within its bound (a key of the state bounds with its own time to live), a seat's outcome that does not read. pubBoundsKV := process.AwaitBucket(ctx, js, conn, versions.BoundsBucket, "a publish no holder answers, and a node holding the old form seven days, are not raised", logger.Printf) + stepReads.bounds, stepReads.findings = func() jetstream.KeyValue { return pubBoundsKV.KV() }, found pubs := &publishes{bounds: pubBoundsKV.KV, versions: numberer.Versions, findings: found, bound: versions.PublishBound, logf: logger.Printf} numberer.Publishing = pubs.arm b := &build.Builder{ @@ -443,6 +447,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro unput.expired(ctx, e) return } + if _, _, ok := versions.StepsBoundOf(e.Key()); ok { + stepReads.expired(ctx, e) + return + } if node, ok := enginereads.BoundOf(e.Key()); ok { if e.Operation() == jetstream.KeyValuePurge { // its five minutes ended reads.expiredCall(node) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepsread.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepsread.go new file mode 100644 index 00000000..378a8db2 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepsread.go @@ -0,0 +1,95 @@ +package main + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + + "mesh-delivery-builder/internal/versions" +) + +// **The builder's read of an ask's steps is a kept call, unanswered past five minutes a finding** (design 47 G §568, A-77, ADR +// 0668 §3): a transport fault opening or taking up the watch of `steps` ends no build key; the read is made again once the bus +// answers. The first fault arms a key of the state bounds (StepsBoundKey) with its own time to live; the bus's marker of its end, +// heard on the watch of the state, raises call-unanswered about the ask while the read is still unanswered; the answer deletes +// the key and clears the finding. No timer. +type stepsRead struct { + bounds func() jetstream.KeyValue + findings *findings + bound time.Duration + now func() time.Time + logf func(string, ...any) + + mu sync.Mutex + open map[string]string // the asks whose read is unanswered, by ask id: the last fault +} + +// unanswered is told each fault of the read of the ask's steps on module: the first arms the bound. +func (s *stepsRead) unanswered(module, askID string, fault error) { + s.mu.Lock() + if s.open == nil { + s.open = map[string]string{} + } + _, armed := s.open[askID] + s.open[askID] = fault.Error() + s.mu.Unlock() + if armed { + return + } + ctx := context.Background() + var err error + if kv := s.bounds(); kv == nil { + err = fmt.Errorf("%s is not on the bus yet", versions.BoundsBucket) + } else if _, err = kv.Create(ctx, versions.StepsBoundKey(module, askID), versions.EncodeBound("the read of the steps of the ask "+askID+" on "+module, s.now().Add(s.bound)), jetstream.KeyTTL(s.bound)); errors.Is(err, jetstream.ErrKeyExists) { + err = nil + } + if err != nil { + s.logf("the bound of the read of the steps of the ask %s is not written, so it is raised now: %v", askID, err) + s.raise(ctx, module, askID, fault.Error(), s.now()) + } +} + +// answered ends the read's bound and clears its finding. +func (s *stepsRead) answered(module, askID string) { + s.mu.Lock() + _, was := s.open[askID] + delete(s.open, askID) + s.mu.Unlock() + if !was { + return + } + ctx := context.Background() + if kv := s.bounds(); kv != nil { + if err := kv.Delete(ctx, versions.StepsBoundKey(module, askID)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + s.logf("the bound of the read of the steps of the ask %s is not deleted: %v", askID, err) + } + } + s.findings.clear(ctx, meshconditionsclient.KindCallUnanswered, meshconditionsclient.ScopeCall, askID) +} + +// expired is told each change of the state bounds: the end of a steps bound raises call-unanswered while its read is unanswered. +func (s *stepsRead) expired(ctx context.Context, e jetstream.KeyValueEntry) { + module, askID, ok := versions.StepsBoundOf(e.Key()) + if !ok || e.Operation() != jetstream.KeyValuePurge { + return + } + s.mu.Lock() + fault, open := s.open[askID] + s.mu.Unlock() + if !open { + return + } + s.raise(ctx, module, askID, fault, s.now().Add(-s.bound)) +} + +func (s *stepsRead) raise(ctx context.Context, module, askID, fault string, since time.Time) { + o := callUnanswered(askID, "the read of the steps of "+module, "", fault, "builder."+meshconditionsclient.KindCallUnanswered, since) + o.needs = "the bus answers the read again by itself; builder_stuck shows the key waiting on the ask." + s.findings.raise(ctx, o) +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepsread_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepsread_test.go new file mode 100644 index 00000000..d47b9c4d --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepsread_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + + "mesh-delivery-builder/internal/versions" +) + +// Design 47 G §568, A-77: the first fault of the read of an ask's steps arms a bound key with its own time to live; its end +// while the read is unanswered raises call-unanswered about the ask; the answer deletes the bound and clears the finding. +func TestTheStepsReadIsCallUnansweredAfterItsBound(t *testing.T) { + st := &memFindings{vals: map[string][]byte{}} + f := &findings{w: st, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }} + marks := &memBounds{keys: map[string]bool{}} + s := &stepsRead{bounds: func() jetstream.KeyValue { return marks }, findings: f, bound: meshconditionsclient.CallUnansweredAfter, + now: func() time.Time { return time.Unix(600, 0) }, logf: t.Logf} + ask := "mvl0frv18d1958" + callKey := meshconditionsclient.Key(meshconditionsclient.ScopeCall, ask, meshconditionsclient.KindCallUnanswered) + s.unanswered("mesh-build-go", ask, errors.New("nats: no responders available for request")) + if !marks.keys[versions.StepsBoundKey("mesh-build-go", ask)] || st.vals[callKey] != nil { + t.Fatalf("a first fault: bound %v, finding %v; want the bound armed, no finding yet", marks.keys, st.vals[callKey] != nil) + } + s.expired(context.Background(), purged{key: versions.StepsBoundKey("mesh-build-go", ask)}) + if got, err := decodeFinding(st.vals[callKey]); err != nil || got.Kind != meshconditionsclient.KindCallUnanswered || !strings.Contains(got.Summary, "the steps of mesh-build-go") || !strings.Contains(got.Summary, ask) { + t.Fatalf("the bound's end: the finding = %+v, %v", got, err) + } + s.answered("mesh-build-go", ask) + if marks.keys[versions.StepsBoundKey("mesh-build-go", ask)] || st.vals[callKey] != nil { + t.Errorf("the read answered: bound %v, finding left %v", marks.keys, st.vals[callKey] != nil) + } + s.expired(context.Background(), purged{key: versions.StepsBoundKey("mesh-build-go", ask)}) + if st.vals[callKey] != nil { + t.Error("a bound's end after the answer raised call-unanswered") + } +} diff --git a/modules/mesh-delivery-builder/internal/versions/publish.go b/modules/mesh-delivery-builder/internal/versions/publish.go index 7b401b8a..43457b3b 100644 --- a/modules/mesh-delivery-builder/internal/versions/publish.go +++ b/modules/mesh-delivery-builder/internal/versions/publish.go @@ -37,6 +37,7 @@ const ( publishBoundPrefix = "publish." oldFormBoundPrefix = "old-form." unputBoundPrefix = "unput." + stepsBoundPrefix = "steps." ) // UnputBoundKey is the bound of a build key a look left building because its end could not be written (issue 1966, @@ -44,6 +45,21 @@ const ( // finding call-unanswered about the key. func UnputBoundKey(buildKey string) string { return unputBoundPrefix + buildKey } +// StepsBoundKey is the bound of the builder's read of an ask's steps while the bus does not answer it (design 47 G §568, A-77): +// `steps..`. Its end, the bus's marker of its time to live, while the read is unanswered, is the finding +// call-unanswered about the ask. +func StepsBoundKey(module, askID string) string { return stepsBoundPrefix + module + "." + askID } + +// StepsBoundOf is the build module and ask a steps bound names, and whether key is one. +func StepsBoundOf(key string) (module, askID string, ok bool) { + rest, ok := strings.CutPrefix(key, stepsBoundPrefix) + if !ok { + return "", "", false + } + module, askID, ok = strings.Cut(rest, ".") + return module, askID, ok && module != "" && askID != "" +} + // UnputBoundOf is the build key an unput bound names, and whether key is one. func UnputBoundOf(key string) (string, bool) { return strings.CutPrefix(key, unputBoundPrefix) }