From f2379dfeea2db27ddda9775c1749a7ba9b0c215d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 09:54:46 +0200 Subject: [PATCH] Add builder_build and builder_builds read tools to the view, so the builder's keys can be checked live (issue 1723) --- .../cmd/mesh-delivery-view-tools/main.go | 37 +++++- .../cmd/mesh-delivery-view-tools/main_test.go | 62 ++++++++++ modules/mesh-delivery-view/go.mod | 2 +- modules/mesh-delivery-view/go.sum | 2 + .../mesh-delivery-view/internal/view/read.go | 112 +++++++++++++++++- modules/mesh-delivery-view/manifest_test.go | 4 +- modules/mesh-delivery-view/module.json | 5 +- 7 files changed, 215 insertions(+), 9 deletions(-) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index ecef8be..b6079de 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -35,18 +35,47 @@ func Tools(r view.Reader) []stdio.Tool { mods, err := r.Modules(only) return map[string]any{"modules": mods}, err }}, + {Name: view.BuilderBuildsReadTool, + Description: "The builder's build keys, each as stored in mesh-delivery-builder_builds: key, module, fingerprint (src2) and build " + + "(state, by, step with the nodes it was built on, manifest, artifacts with their digests, toolchains, error). Narrow by module " + + "or state (queued, building, done, failed). Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "only this module's builds"}, + "state": map[string]any{"type": "string", "description": "only builds in this state"}}}, + Run: func(args map[string]any) (any, error) { + module, _ := args["module"].(string) + state, _ := args["state"].(string) + rows, err := r.BuilderBuilds(module, state) + return map[string]any{"builds": rows}, err + }}, + {Name: view.BuilderBuildReadTool, + Description: "One build of the builder as stored. Give the module and its fingerprint (fp2, src2:...), or the module alone for the build " + + "at the fingerprint the indexer holds for it now, joined through modules.; with commit, only if the indexer holds the module at that commit. Read-only.", + Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module's name"}, + "fp2": map[string]any{"type": "string", "description": "its new-form fingerprint; empty means the indexer's current one"}, + "commit": map[string]any{"type": "string", "description": "the commit the indexer must hold the module at"}}}, + Run: func(args map[string]any) (any, error) { + module, _ := args["module"].(string) + fp2, _ := args["fp2"].(string) + commit, _ := args["commit"].(string) + return r.BuilderBuild(module, fp2, commit) + }}, } } -func main() { - state := stdio.State("mesh-delivery-indexer.modules") - r := view.Reader{State: view.StateRead{Keys: state.Keys, Get: func(key string) (json.RawMessage, error) { +func read(state stdio.KeptState) view.StateRead { + return view.StateRead{Keys: state.Keys, Get: func(key string) (json.RawMessage, error) { e, err := state.Get(key) if err != nil || e == nil { return nil, err } return e.Value, nil - }}} + }} +} + +func main() { + r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds"))} if err := stdio.Serve("", Tools(r)); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go index bfa3124..a5e2193 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go @@ -94,3 +94,65 @@ func TestModulesToolAnswersEveryKeyAsStored(t *testing.T) { t.Errorf("none2 modules = %v", only) } } + +const ( + fp2A = "src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + fp2B = "src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + keyA = "netcheck.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + keyB = "zsh.bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + doneBuild = `{"v":1,"state":"done","by":"novox.mesh-delivery-builder","step":"built on shanks, ace","manifest":"m","artifacts":[{"name":"program","kind":"bundle","digest":"sha256:d1"}]}` + failBuild = `{"v":1,"state":"failed","by":"novox.mesh-delivery-builder","error":"refused kind: package"}` +) + +func both(builds, modules fake) view.Reader { + r := modules.reader() + r.Builds = builds.reader().State + return r +} + +// 1723/AC1: builder_builds lists every key as stored (module, fingerprint, state, digests, error), and narrows by module and state. +func TestBuilderBuildsListsKeysAsStored(t *testing.T) { + r := both(fake{keyA: doneBuild, keyB: failBuild, "junk": "{}"}, fake{}) + out, err := run(t, r, view.BuilderBuildsReadTool, map[string]any{}) + if err != nil { + t.Fatal(err) + } + rows := out.(map[string]any)["builds"].([]view.BuilderRow) + if len(rows) != 2 || rows[0].Key != keyA || rows[0].Module != "netcheck" || rows[0].Fingerprint != fp2A || string(rows[0].Build) != doneBuild || string(rows[1].Build) != failBuild { + t.Errorf("rows = %+v", rows) + } + for args, want := range map[string]string{"state": keyB, "module": keyA} { + out, _ := run(t, r, view.BuilderBuildsReadTool, map[string]any{args: map[string]string{keyB: "failed", keyA: "netcheck"}[want]}) + got := out.(map[string]any)["builds"].([]view.BuilderRow) + if len(got) != 1 || got[0].Key != want { + t.Errorf("%s filter = %+v, want %s", args, got, want) + } + } +} + +// 1723/AC2: builder_build answers one key by module and fp2, and by module alone through the indexer's fp2 (joined at its commit). +func TestBuilderBuildJoinsTheIndexersFingerprint(t *testing.T) { + idx := `{"v":3,"fp2":"` + fp2A + `","commit":"c1","path":"."}` + r := both(fake{keyA: doneBuild}, fake{"netcheck": idx, "zsh": `{"v":3,"none2":"not known","commit":"c9"}`}) + for _, args := range []map[string]any{{"module": "netcheck", "fp2": fp2A}, {"module": "netcheck"}, {"module": "netcheck", "commit": "c1"}} { + out, err := run(t, r, view.BuilderBuildReadTool, args) + if err != nil { + t.Fatalf("%v: %v", args, err) + } + if row := out.(view.BuilderRow); row.Key != keyA || string(row.Build) != doneBuild { + t.Errorf("%v = %+v", args, row) + } + } + for args, want := range map[string]string{"netcheck/c2": "not c2", "zsh": "not known", "gone": "no module gone"} { + a := map[string]any{"module": strings.Split(args, "/")[0]} + if strings.Contains(args, "/") { + a["commit"] = "c2" + } + if _, err := run(t, r, view.BuilderBuildReadTool, a); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("%v: err = %v, want %q", a, err, want) + } + } + if _, err := run(t, r, view.BuilderBuildReadTool, map[string]any{"module": "netcheck", "fp2": fp2B}); err == nil || !strings.Contains(err.Error(), "holds no build") { + t.Errorf("an unbuilt fingerprint: %v", err) + } +} diff --git a/modules/mesh-delivery-view/go.mod b/modules/mesh-delivery-view/go.mod index 9ca3744..b8f5dd8 100644 --- a/modules/mesh-delivery-view/go.mod +++ b/modules/mesh-delivery-view/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-view go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-view/go.sum b/modules/mesh-delivery-view/go.sum index 962db0d..3686111 100644 --- a/modules/mesh-delivery-view/go.sum +++ b/modules/mesh-delivery-view/go.sum @@ -6,6 +6,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69 h1:kvtV/SPs git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef h1:xy3wuGWJpqMAXYdXolocYG6w9xnWqBltVUffoaziIKo= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/internal/view/read.go b/modules/mesh-delivery-view/internal/view/read.go index 05e547e..e99a327 100644 --- a/modules/mesh-delivery-view/internal/view/read.go +++ b/modules/mesh-delivery-view/internal/view/read.go @@ -26,7 +26,10 @@ type StateRead struct { } // Reader answers the read tools from a modules state. -type Reader struct{ State StateRead } +type Reader struct { + State StateRead // the indexer's modules + Builds StateRead // the builder's builds +} // Module is the stored value of modules., exactly as the indexer wrote it. func (r Reader) Module(module string) (json.RawMessage, error) { @@ -80,3 +83,110 @@ func (r Reader) Modules(none2 bool) (map[string]json.RawMessage, error) { } return out, nil } + +// The read tools of the builder's builds (novox/hq issue 1723): what the builder stored under `.` +// in mesh-delivery-builder_builds, answered as stored, so its states, nodes (in the step) and digests can be checked live. +const ( + BuilderBuildReadTool = "builder_build" + BuilderBuildsReadTool = "builder_builds" +) + +// BuilderRow is one key of the builder's builds: the key, the module and the fingerprint it parses to, and the stored value +// as the builder wrote it (state, by, step, manifest, artifacts with digests, toolchains, error). +type BuilderRow struct { + Key string `json:"key"` + Module string `json:"module"` + Fingerprint string `json:"fingerprint"` + Build json.RawMessage `json:"build"` +} + +func (r Reader) builderRow(key string) (BuilderRow, bool, error) { + module, fp, err := delivery.ParseBuilderBuildKey(key) + if err != nil { + return BuilderRow{}, false, nil // a key that is no build key is not a build + } + raw, err := r.Builds.Get(key) + if err != nil { + return BuilderRow{}, false, err + } + if len(raw) == 0 { + return BuilderRow{}, false, nil + } + return BuilderRow{Key: key, Module: module, Fingerprint: fp, Build: raw}, true, nil +} + +// BuilderBuilds lists the builder's keys, each as stored, sorted by key; module and state, when given, keep only the builds +// of that module or in that state. +func (r Reader) BuilderBuilds(module, state string) ([]BuilderRow, error) { + if r.Builds.Keys == nil || r.Builds.Get == nil { + return nil, fmt.Errorf("the builder's builds cannot be read here") + } + keys, err := r.Builds.Keys() + if err != nil { + return nil, err + } + sort.Strings(keys) + rows := []BuilderRow{} + for _, k := range keys { + row, ok, err := r.builderRow(k) + if err != nil { + return nil, err + } + if !ok || (module != "" && row.Module != module) { + continue + } + if state != "" { + var b struct { + State string `json:"state"` + } + if json.Unmarshal(row.Build, &b) != nil || b.State != state { + continue + } + } + rows = append(rows, row) + } + return rows, nil +} + +// BuilderBuild is the build of module at the new-form fingerprint fp2; with an empty fp2 it is the build at the fingerprint +// the indexer holds for the module now (its fp2), joined through modules.; commit, when given, must be the commit +// the indexer holds that fp2 for, or the answer says which commit it holds (the fingerprint of another commit is not guessed). +func (r Reader) BuilderBuild(module, fp2, commit string) (BuilderRow, error) { + module = strings.TrimSpace(module) + if r.Builds.Get == nil { + return BuilderRow{}, fmt.Errorf("the builder's builds cannot be read here") + } + if fp2 == "" { + raw, err := r.Module(module) + if err != nil { + return BuilderRow{}, err + } + var m struct { + FP2 string `json:"fp2"` + None2 string `json:"none2"` + Commit string `json:"commit"` + } + if err := json.Unmarshal(raw, &m); err != nil { + return BuilderRow{}, fmt.Errorf("the indexer's value of %s is not readable: %w", module, err) + } + if commit != "" && m.Commit != commit { + return BuilderRow{}, fmt.Errorf("the indexer holds %s at commit %s, not %s, so its fingerprint is not the one of that commit", module, m.Commit, commit) + } + if m.FP2 == "" { + return BuilderRow{}, fmt.Errorf("the indexer holds no new-form fingerprint of %s: %s", module, m.None2) + } + fp2 = m.FP2 + } + key, err := delivery.BuilderBuildKey(module, fp2) + if err != nil { + return BuilderRow{}, err + } + row, ok, err := r.builderRow(key) + if err != nil { + return BuilderRow{}, err + } + if !ok { + return BuilderRow{}, fmt.Errorf("the builder holds no build %s", key) + } + return row, nil +} diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index aa54e36..0e185a2 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -76,7 +76,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 if m.Module != "mesh-delivery-view" { t.Errorf("module = %q", m.Module) } - want := []string{"mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-gate.asks", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // 1168/AC1, 1241/AC4, 1345/AC3 + want := []string{"mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-builder.builds", "mesh-delivery-gate.asks", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // 1168/AC1, 1241/AC4, 1345/AC3 got := slices.Clone(m.Reads) slices.Sort(got) if !reflect.DeepEqual(got, want) { @@ -95,7 +95,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 } // The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer // itself serves no tool (ADR 0437 ยง4). Carrying them gives the tool runner read access to every bucket the view reads. - if want := []string{"indexer_module", "indexer_modules"}; !reflect.DeepEqual(m.Tools, want) { + if want := []string{"indexer_module", "indexer_modules", "builder_build", "builder_builds"}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } for _, a := range m.Build.Artifacts { diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index dc14e37..89bf62c 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -5,11 +5,14 @@ "slug": "view", "tools": [ "indexer_module", - "indexer_modules" + "indexer_modules", + "builder_build", + "builder_builds" ], "reads": [ "mesh-controller.assignments", "mesh-controller.builds", + "mesh-delivery-builder.builds", "mesh-delivery-gate.asks", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules",