diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index b6079de..645c726 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -35,18 +35,39 @@ func Tools(r view.Reader) []stdio.Tool { mods, err := r.Modules(only) return map[string]any{"modules": mods}, err }}, + {Name: view.CandidateReadTool, + Description: "What the indexer stored for one candidate (the open pull requests of a branch) under candidates., as stored: " + + "the pull requests and heads, the base, and per pull request the touched modules with their new-form fingerprint (fp2) or why " + + "they have none (none2). Read-only.", + Input: map[string]any{"type": "object", "required": []string{"group"}, "properties": map[string]any{ + "group": map[string]any{"type": "string", "description": "the candidate's group, the branch name its pull requests share"}}}, + Run: func(args map[string]any) (any, error) { + group, _ := args["group"].(string) + return r.Candidate(group) + }}, + {Name: view.CandidatesReadTool, + Description: "Every candidate the indexer holds, by group, each as stored under candidates.; with module, only those with a " + + "pull request touching that module. Keys that cannot be read are listed as unreadable. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "only candidates touching this module"}}}, + Run: func(args map[string]any) (any, error) { + module, _ := args["module"].(string) + c, bad, err := r.Candidates(module) + return map[string]any{"candidates": c, "unreadable": bad}, err + }}, {Name: view.BuilderBuildsReadTool, Description: "The builder's build keys, each as stored in mesh-delivery-builder_builds: key, module, fingerprint (src2) and build " + "(state, by, step with the nodes it was built on, manifest, artifacts with their digests, toolchains, error). Narrow by module " + - "or state (queued, building, done, failed). Read-only.", + "or state (queued, building, done, failed); at most limit builds (default 50), newest revision first for a module, and the answer says when it was cut and lists keys that are no build key (unreadable). Without a module, give one to get its newest. Read-only.", Input: map[string]any{"type": "object", "properties": map[string]any{ "module": map[string]any{"type": "string", "description": "only this module's builds"}, - "state": map[string]any{"type": "string", "description": "only builds in this state"}}}, + "state": map[string]any{"type": "string", "description": "only builds in this state"}, + "limit": map[string]any{"type": "integer", "description": "at most this many builds (default 50), newest revision first"}}}, Run: func(args map[string]any) (any, error) { module, _ := args["module"].(string) state, _ := args["state"].(string) - rows, err := r.BuilderBuilds(module, state) - return map[string]any{"builds": rows}, err + limit, _ := args["limit"].(float64) + return r.BuilderBuilds(module, state, int(limit)) }}, {Name: view.BuilderBuildReadTool, Description: "One build of the builder as stored. Give the module and its fingerprint (fp2, src2:...), or the module alone for the build " + @@ -65,7 +86,13 @@ func Tools(r view.Reader) []stdio.Tool { } func read(state stdio.KeptState) view.StateRead { - return view.StateRead{Keys: state.Keys, Get: func(key string) (json.RawMessage, error) { + return view.StateRead{Keys: state.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) { + e, err := state.Get(key) + if err != nil || e == nil { + return nil, 0, err + } + return e.Value, e.Revision, nil + }, Get: func(key string) (json.RawMessage, error) { e, err := state.Get(key) if err != nil || e == nil { return nil, err @@ -75,7 +102,7 @@ func read(state stdio.KeptState) view.StateRead { } func main() { - r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds"))} + r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates"))} if err := stdio.Serve("", Tools(r)); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go index a5e2193..0c13bdf 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "fmt" "strings" "testing" @@ -117,13 +118,16 @@ func TestBuilderBuildsListsKeysAsStored(t *testing.T) { if err != nil { t.Fatal(err) } - rows := out.(map[string]any)["builds"].([]view.BuilderRow) + rows := out.(view.BuildsAnswer).Builds + if u := out.(view.BuildsAnswer).Unreadable; len(u) != 1 || u[0] != "junk" { + t.Errorf("unreadable = %v, want [junk] (issue 1818)", u) + } if len(rows) != 2 || rows[0].Key != keyA || rows[0].Module != "netcheck" || rows[0].Fingerprint != fp2A || string(rows[0].Build) != doneBuild || string(rows[1].Build) != failBuild { t.Errorf("rows = %+v", rows) } for args, want := range map[string]string{"state": keyB, "module": keyA} { out, _ := run(t, r, view.BuilderBuildsReadTool, map[string]any{args: map[string]string{keyB: "failed", keyA: "netcheck"}[want]}) - got := out.(map[string]any)["builds"].([]view.BuilderRow) + got := out.(view.BuildsAnswer).Builds if len(got) != 1 || got[0].Key != want { t.Errorf("%s filter = %+v, want %s", args, got, want) } @@ -156,3 +160,107 @@ func TestBuilderBuildJoinsTheIndexersFingerprint(t *testing.T) { t.Errorf("an unbuilt fingerprint: %v", err) } } + +// bigBuilds is a bucket of n keys over 100 modules that counts the values read. +type bigBuilds struct { + fake fake + reads *int +} + +func newBig(n int) bigBuilds { + f := fake{} + for i := 0; i < n; i++ { + f[fmt.Sprintf("mod%02d.%064x", i%100, i)] = doneBuild + } + return bigBuilds{fake: f, reads: new(int)} +} + +func (b bigBuilds) reader() view.Reader { + r := b.fake.reader() + get := r.State.Get + r.Builds = view.StateRead{Keys: r.State.Keys, Get: func(k string) (json.RawMessage, error) { *b.reads++; return get(k) }, + GetEntry: func(k string) (json.RawMessage, uint64, error) { + *b.reads++ + raw, err := get(k) + var rev uint64 + fmt.Sscanf(k[strings.Index(k, ".")+1:], "%x", &rev) // the fake's revision is the key's number + return raw, rev + 1, err + }} + return r +} + +// 1827/AC1: with 2,000 keys, a module's builds are found by key prefix, so only that module's values are read (20, not 2,000), +// newest revision first, within the limit, and the answer says it was cut. +func TestBuilderBuildsFiltersByPrefixBeforeAnyGet(t *testing.T) { + b := newBig(2000) + out, err := run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{"module": "mod07", "limit": float64(5)}) + if err != nil { + t.Fatal(err) + } + a := out.(view.BuildsAnswer) + if *b.reads != 20 { + t.Errorf("read %d values for a module of 20 keys, want 20", *b.reads) + } + if len(a.Builds) != 5 || !a.Cut || !strings.Contains(a.Note, "5") { + t.Errorf("answer = %d rows, cut %v, note %q", len(a.Builds), a.Cut, a.Note) + } + for i := 1; i < len(a.Builds); i++ { + if a.Builds[i-1].Revision < a.Builds[i].Revision { + t.Errorf("not newest first: %d before %d", a.Builds[i-1].Revision, a.Builds[i].Revision) + } + } + if a.Builds[0].Revision != 1908 { + t.Errorf("newest revision = %d, want 1908", a.Builds[0].Revision) + } +} + +// 1827/AC2: with no module the answer reads at most the limit's worth in key order (default 50), never the 2,000. +func TestBuilderBuildsWithoutAModuleIsBounded(t *testing.T) { + b := newBig(2000) + out, err := run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{}) + if err != nil { + t.Fatal(err) + } + a := out.(view.BuildsAnswer) + if len(a.Builds) != view.DefaultBuildsLimit || !a.Cut || *b.reads > view.DefaultBuildsLimit { + t.Errorf("%d rows, cut %v, %d reads", len(a.Builds), a.Cut, *b.reads) + } + b = newBig(2000) + out, _ = run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{"state": "failed"}) + if a := out.(view.BuildsAnswer); *b.reads != view.MaxBuildReads || !a.Cut || len(a.Builds) != 0 { + t.Errorf("a state matching nothing: %d reads (want the bound %d), cut %v", *b.reads, view.MaxBuildReads, a.Cut) + } +} + +// 1818: fp2 together with commit is refused, naming both. +func TestBuilderBuildRefusesFP2WithCommit(t *testing.T) { + r := both(fake{keyA: doneBuild}, fake{"netcheck": `{"v":3,"fp2":"` + fp2A + `","commit":"c1","path":"."}`}) + _, err := run(t, r, view.BuilderBuildReadTool, map[string]any{"module": "netcheck", "fp2": fp2A, "commit": "c1"}) + if err == nil || !strings.Contains(err.Error(), "fp2") || !strings.Contains(err.Error(), "commit") || !strings.Contains(err.Error(), "not both") { + t.Errorf("err = %v", err) + } +} + +// 1681: indexer_candidates and indexer_candidate answer the stored value; a pull request's fp2 per touched module shows. +func TestCandidateToolsAnswerTheStoredValue(t *testing.T) { + cand := `{"v":1,"group":"feat/x","base":3,"heads":"h","pulls":[{"modules":["zsh"],"fps":{"zsh":{"fp2":"` + fp2B + `","manifest":"m"}}}]}` + other := `{"v":1,"group":"fix","base":3,"heads":"h","pulls":[{"modules":["netcheck"]}]}` + r := fake{"feat/x": cand, "fix": other, "bad": "{"}.reader() + r.CandidatesState = r.State + out, err := run(t, r, view.CandidateReadTool, map[string]any{"group": "feat/x"}) + if err != nil || string(out.(json.RawMessage)) != cand || !strings.Contains(string(out.(json.RawMessage)), fp2B) { + t.Fatalf("candidate = %v, %v", out, err) + } + if _, err := run(t, r, view.CandidateReadTool, map[string]any{"group": "gone"}); err == nil || !strings.Contains(err.Error(), "no candidate gone") { + t.Errorf("err = %v", err) + } + out, err = run(t, r, view.CandidatesReadTool, map[string]any{}) + m := out.(map[string]any) + if err != nil || len(m["candidates"].(map[string]json.RawMessage)) != 2 || len(m["unreadable"].([]string)) != 1 { + t.Errorf("all = %v, %v", m, err) + } + out, _ = run(t, r, view.CandidatesReadTool, map[string]any{"module": "zsh"}) + if c := out.(map[string]any)["candidates"].(map[string]json.RawMessage); len(c) != 1 || c["feat/x"] == nil { + t.Errorf("by module = %v", c) + } +} diff --git a/modules/mesh-delivery-view/internal/view/read.go b/modules/mesh-delivery-view/internal/view/read.go index e99a327..8f1546a 100644 --- a/modules/mesh-delivery-view/internal/view/read.go +++ b/modules/mesh-delivery-view/internal/view/read.go @@ -7,6 +7,7 @@ package view import ( "encoding/json" "fmt" + "slices" "sort" "strings" @@ -23,12 +24,16 @@ const ( type StateRead struct { Keys func() ([]string, error) Get func(key string) (json.RawMessage, error) + // GetEntry, when set, also gives the revision of the value; builder_builds orders by it. + GetEntry func(key string) (json.RawMessage, uint64, error) } // Reader answers the read tools from a modules state. type Reader struct { State StateRead // the indexer's modules Builds StateRead // the builder's builds + + CandidatesState StateRead // the indexer's candidates } // Module is the stored value of modules., exactly as the indexer wrote it. @@ -84,6 +89,67 @@ func (r Reader) Modules(none2 bool) (map[string]json.RawMessage, error) { return out, nil } +// The read tools of the indexer's candidates (issue 1681): what the indexer stored under candidates., as stored, so the +// new-form fingerprint of each touched module of an open pull request (pulls[].fps) can be read live. +const ( + CandidateReadTool = "indexer_candidate" + CandidatesReadTool = "indexer_candidates" +) + +// Candidate is the stored value of the candidates key of a group, exactly as the indexer wrote it. +func (r Reader) Candidate(group string) (json.RawMessage, error) { + key, err := delivery.CandidatesKey(strings.TrimSpace(group)) + if err != nil { + return nil, err + } + if r.CandidatesState.Get == nil { + return nil, fmt.Errorf("the candidates state cannot be read here") + } + raw, err := r.CandidatesState.Get(key) + if err != nil { + return nil, err + } + if len(raw) == 0 { + return nil, fmt.Errorf("no candidate %s in the candidates state", group) + } + return raw, nil +} + +// Candidates is every candidate, by group, each as stored; with a module only the candidates with a pull request touching it. +// A value that is not readable is listed under unreadable with its key, not skipped. +func (r Reader) Candidates(module string) (map[string]json.RawMessage, []string, error) { + if r.CandidatesState.Keys == nil || r.CandidatesState.Get == nil { + return nil, nil, fmt.Errorf("the candidates state cannot be read here") + } + keys, err := r.CandidatesState.Keys() + if err != nil { + return nil, nil, err + } + sort.Strings(keys) + out := map[string]json.RawMessage{} + var unreadable []string + for _, k := range keys { + raw, err := r.CandidatesState.Get(k) + if err != nil { + return nil, nil, err + } + if len(raw) == 0 { + continue + } + group, perr := delivery.ParseCandidatesKey(k) + var c delivery.Candidate + if perr != nil || json.Unmarshal(raw, &c) != nil { + unreadable = append(unreadable, k) + continue + } + if module != "" && !slices.ContainsFunc(c.Pulls, func(p delivery.CandidatePull) bool { return slices.Contains(p.Modules, module) }) { + continue + } + out[group] = raw + } + return out, unreadable, nil +} + // The read tools of the builder's builds (novox/hq issue 1723): what the builder stored under `.` // in mesh-delivery-builder_builds, answered as stored, so its states, nodes (in the step) and digests can be checked live. const ( @@ -91,48 +157,100 @@ const ( BuilderBuildsReadTool = "builder_builds" ) -// BuilderRow is one key of the builder's builds: the key, the module and the fingerprint it parses to, and the stored value -// as the builder wrote it (state, by, step, manifest, artifacts with digests, toolchains, error). +// BuilderRow is one key of the builder's builds: the key, the module and the fingerprint it parses to, the revision of its +// stored value (when the read gives one) and the stored value as the builder wrote it. type BuilderRow struct { Key string `json:"key"` Module string `json:"module"` Fingerprint string `json:"fingerprint"` + Revision uint64 `json:"revision,omitempty"` Build json.RawMessage `json:"build"` } +// The bounds of builder_builds (issue 1827): the rows it returns when no limit is given, and the values it reads in one call. +// Reading a value is a bus round trip, so the answer must not read the whole bucket (it timed out at 30 s on the live one). +const ( + DefaultBuildsLimit = 50 + MaxBuildReads = 200 +) + +// BuildsAnswer is the answer of builder_builds: the rows, newest revision first, whether the answer was cut and why, and +// the keys that are no build key (Unreadable), which are listed rather than silently skipped (issue 1818). +type BuildsAnswer struct { + Builds []BuilderRow `json:"builds"` + Cut bool `json:"cut,omitempty"` + Note string `json:"note,omitempty"` + Unreadable []string `json:"unreadable,omitempty"` +} + +func (r Reader) get(key string) (json.RawMessage, uint64, error) { + if r.Builds.GetEntry != nil { + return r.Builds.GetEntry(key) + } + raw, err := r.Builds.Get(key) + return raw, 0, err +} + func (r Reader) builderRow(key string) (BuilderRow, bool, error) { module, fp, err := delivery.ParseBuilderBuildKey(key) if err != nil { return BuilderRow{}, false, nil // a key that is no build key is not a build } - raw, err := r.Builds.Get(key) + raw, rev, err := r.get(key) if err != nil { return BuilderRow{}, false, err } if len(raw) == 0 { return BuilderRow{}, false, nil } - return BuilderRow{Key: key, Module: module, Fingerprint: fp, Build: raw}, true, nil + return BuilderRow{Key: key, Module: module, Fingerprint: fp, Revision: rev, Build: raw}, true, nil } -// BuilderBuilds lists the builder's keys, each as stored, sorted by key; module and state, when given, keep only the builds -// of that module or in that state. -func (r Reader) BuilderBuilds(module, state string) ([]BuilderRow, error) { +// BuilderBuilds lists the builder's keys, each as stored, newest revision first. With a module it considers only the keys +// that begin `.`, before any value is read; with a state it keeps the builds in that state. It returns at most limit +// rows (DefaultBuildsLimit when limit is not above zero) and reads at most MaxBuildReads values, and says when it cut the +// answer. Keys that are no build key are listed as unreadable. Without a module the keys are walked in key order, so the +// answer is the first rows found, not the newest of the bucket: give a module to get its newest. +func (r Reader) BuilderBuilds(module, state string, limit int) (BuildsAnswer, error) { if r.Builds.Keys == nil || r.Builds.Get == nil { - return nil, fmt.Errorf("the builder's builds cannot be read here") + return BuildsAnswer{}, fmt.Errorf("the builder's builds cannot be read here") + } + if limit <= 0 { + limit = DefaultBuildsLimit } keys, err := r.Builds.Keys() if err != nil { - return nil, err + return BuildsAnswer{}, err } sort.Strings(keys) - rows := []BuilderRow{} + prefix := "" + if module = strings.TrimSpace(module); module != "" { + prefix = module + "." + } + out := BuildsAnswer{Builds: []BuilderRow{}} + var candidates []string for _, k := range keys { + if prefix != "" && !strings.HasPrefix(k, prefix) { + continue + } + if _, _, err := delivery.ParseBuilderBuildKey(k); err != nil { + out.Unreadable = append(out.Unreadable, k) + continue + } + candidates = append(candidates, k) + } + reads := 0 + for _, k := range candidates { + if reads >= MaxBuildReads { + out.Cut, out.Note = true, fmt.Sprintf("read %d values and stopped; give a module (and a state) to narrow the answer", MaxBuildReads) + break + } + reads++ row, ok, err := r.builderRow(k) if err != nil { - return nil, err + return BuildsAnswer{}, err } - if !ok || (module != "" && row.Module != module) { + if !ok { continue } if state != "" { @@ -143,9 +261,26 @@ func (r Reader) BuilderBuilds(module, state string) ([]BuilderRow, error) { continue } } - rows = append(rows, row) + out.Builds = append(out.Builds, row) + // Without a module the walk is in key order and stops at the limit; with one, all its (few) keys are read to order them. + if prefix == "" && len(out.Builds) >= limit { + if reads < len(candidates) { + out.Cut, out.Note = true, fmt.Sprintf("stopped at the limit of %d builds in key order, not newest first; give a module for its newest", limit) + } + break + } } - return rows, nil + sort.SliceStable(out.Builds, func(i, j int) bool { + if out.Builds[i].Revision != out.Builds[j].Revision { + return out.Builds[i].Revision > out.Builds[j].Revision + } + return out.Builds[i].Key < out.Builds[j].Key + }) + if len(out.Builds) > limit { + out.Builds = out.Builds[:limit] + out.Cut, out.Note = true, fmt.Sprintf("cut to the newest %d builds of %s", limit, module) + } + return out, nil } // BuilderBuild is the build of module at the new-form fingerprint fp2; with an empty fp2 it is the build at the fingerprint @@ -156,6 +291,9 @@ func (r Reader) BuilderBuild(module, fp2, commit string) (BuilderRow, error) { if r.Builds.Get == nil { return BuilderRow{}, fmt.Errorf("the builder's builds cannot be read here") } + if fp2 != "" && commit != "" { + return BuilderRow{}, fmt.Errorf("give fp2 or commit, not both: fp2 %s names a build directly, commit %s asks for the build at the indexer's fingerprint of that commit", fp2, commit) + } if fp2 == "" { raw, err := r.Module(module) if err != nil { diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index 0e185a2..b3c0e1c 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -95,7 +95,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 } // The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer // itself serves no tool (ADR 0437 ยง4). Carrying them gives the tool runner read access to every bucket the view reads. - if want := []string{"indexer_module", "indexer_modules", "builder_build", "builder_builds"}; !reflect.DeepEqual(m.Tools, want) { + if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds"}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } for _, a := range m.Build.Artifacts { diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index 89bf62c..09cd55b 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -6,6 +6,8 @@ "tools": [ "indexer_module", "indexer_modules", + "indexer_candidate", + "indexer_candidates", "builder_build", "builder_builds" ],