Bound builder_builds by key prefix, limit and read budget, list unreadable keys, refuse fp2 with commit, add candidate reads (issue 1827 V1-V3, 1818, 1681)

modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go

@@ -35,18 +35,39 @@ func Tools(r view.Reader) []stdio.Tool {
 				mods, err := r.Modules(only)
 				return map[string]any{"modules": mods}, err
 			}},
+		{Name: view.CandidateReadTool,
+			Description: "What the indexer stored for one candidate (the open pull requests of a branch) under candidates.<group>, as stored: " +
+				"the pull requests and heads, the base, and per pull request the touched modules with their new-form fingerprint (fp2) or why " +
+				"they have none (none2). Read-only.",
+			Input: map[string]any{"type": "object", "required": []string{"group"}, "properties": map[string]any{
+				"group": map[string]any{"type": "string", "description": "the candidate's group, the branch name its pull requests share"}}},
+			Run: func(args map[string]any) (any, error) {
+				group, _ := args["group"].(string)
+				return r.Candidate(group)
+			}},
+		{Name: view.CandidatesReadTool,
+			Description: "Every candidate the indexer holds, by group, each as stored under candidates.<group>; with module, only those with a " +
+				"pull request touching that module. Keys that cannot be read are listed as unreadable. Read-only.",
+			Input: map[string]any{"type": "object", "properties": map[string]any{
+				"module": map[string]any{"type": "string", "description": "only candidates touching this module"}}},
+			Run: func(args map[string]any) (any, error) {
+				module, _ := args["module"].(string)
+				c, bad, err := r.Candidates(module)
+				return map[string]any{"candidates": c, "unreadable": bad}, err
+			}},
 		{Name: view.BuilderBuildsReadTool,
 			Description: "The builder's build keys, each as stored in mesh-delivery-builder_builds: key, module, fingerprint (src2) and build " +
 				"(state, by, step with the nodes it was built on, manifest, artifacts with their digests, toolchains, error). Narrow by module " +
-				"or state (queued, building, done, failed). Read-only.",
+				"or state (queued, building, done, failed); at most limit builds (default 50), newest revision first for a module, and the answer says when it was cut and lists keys that are no build key (unreadable). Without a module, give one to get its newest. Read-only.",
 			Input: map[string]any{"type": "object", "properties": map[string]any{
 				"module": map[string]any{"type": "string", "description": "only this module's builds"},
-				"state":  map[string]any{"type": "string", "description": "only builds in this state"}}},
+				"state":  map[string]any{"type": "string", "description": "only builds in this state"},
+				"limit":  map[string]any{"type": "integer", "description": "at most this many builds (default 50), newest revision first"}}},
 			Run: func(args map[string]any) (any, error) {
 				module, _ := args["module"].(string)
 				state, _ := args["state"].(string)
-				rows, err := r.BuilderBuilds(module, state)
-				return map[string]any{"builds": rows}, err
+				limit, _ := args["limit"].(float64)
+				return r.BuilderBuilds(module, state, int(limit))
 			}},
 		{Name: view.BuilderBuildReadTool,
 			Description: "One build of the builder as stored. Give the module and its fingerprint (fp2, src2:...), or the module alone for the build " +
@@ -65,7 +86,13 @@ func Tools(r view.Reader) []stdio.Tool {
 }
 
 func read(state stdio.KeptState) view.StateRead {
-	return view.StateRead{Keys: state.Keys, Get: func(key string) (json.RawMessage, error) {
+	return view.StateRead{Keys: state.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) {
+		e, err := state.Get(key)
+		if err != nil || e == nil {
+			return nil, 0, err
+		}
+		return e.Value, e.Revision, nil
+	}, Get: func(key string) (json.RawMessage, error) {
 		e, err := state.Get(key)
 		if err != nil || e == nil {
 			return nil, err
@@ -75,7 +102,7 @@ func read(state stdio.KeptState) view.StateRead {
 }
 
 func main() {
-	r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds"))}
+	r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates"))}
 	if err := stdio.Serve("", Tools(r)); err != nil {
 		fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err)
 		os.Exit(1)

modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
 	"strings"
 	"testing"
 
@@ -117,13 +118,16 @@ func TestBuilderBuildsListsKeysAsStored(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	rows := out.(map[string]any)["builds"].([]view.BuilderRow)
+	rows := out.(view.BuildsAnswer).Builds
+	if u := out.(view.BuildsAnswer).Unreadable; len(u) != 1 || u[0] != "junk" {
+		t.Errorf("unreadable = %v, want [junk] (issue 1818)", u)
+	}
 	if len(rows) != 2 || rows[0].Key != keyA || rows[0].Module != "netcheck" || rows[0].Fingerprint != fp2A || string(rows[0].Build) != doneBuild || string(rows[1].Build) != failBuild {
 		t.Errorf("rows = %+v", rows)
 	}
 	for args, want := range map[string]string{"state": keyB, "module": keyA} {
 		out, _ := run(t, r, view.BuilderBuildsReadTool, map[string]any{args: map[string]string{keyB: "failed", keyA: "netcheck"}[want]})
-		got := out.(map[string]any)["builds"].([]view.BuilderRow)
+		got := out.(view.BuildsAnswer).Builds
 		if len(got) != 1 || got[0].Key != want {
 			t.Errorf("%s filter = %+v, want %s", args, got, want)
 		}
@@ -156,3 +160,107 @@ func TestBuilderBuildJoinsTheIndexersFingerprint(t *testing.T) {
 		t.Errorf("an unbuilt fingerprint: %v", err)
 	}
 }
+
+// bigBuilds is a bucket of n keys over 100 modules that counts the values read.
+type bigBuilds struct {
+	fake  fake
+	reads *int
+}
+
+func newBig(n int) bigBuilds {
+	f := fake{}
+	for i := 0; i < n; i++ {
+		f[fmt.Sprintf("mod%02d.%064x", i%100, i)] = doneBuild
+	}
+	return bigBuilds{fake: f, reads: new(int)}
+}
+
+func (b bigBuilds) reader() view.Reader {
+	r := b.fake.reader()
+	get := r.State.Get
+	r.Builds = view.StateRead{Keys: r.State.Keys, Get: func(k string) (json.RawMessage, error) { *b.reads++; return get(k) },
+		GetEntry: func(k string) (json.RawMessage, uint64, error) {
+			*b.reads++
+			raw, err := get(k)
+			var rev uint64
+			fmt.Sscanf(k[strings.Index(k, ".")+1:], "%x", &rev) // the fake's revision is the key's number
+			return raw, rev + 1, err
+		}}
+	return r
+}
+
+// 1827/AC1: with 2,000 keys, a module's builds are found by key prefix, so only that module's values are read (20, not 2,000),
+// newest revision first, within the limit, and the answer says it was cut.
+func TestBuilderBuildsFiltersByPrefixBeforeAnyGet(t *testing.T) {
+	b := newBig(2000)
+	out, err := run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{"module": "mod07", "limit": float64(5)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	a := out.(view.BuildsAnswer)
+	if *b.reads != 20 {
+		t.Errorf("read %d values for a module of 20 keys, want 20", *b.reads)
+	}
+	if len(a.Builds) != 5 || !a.Cut || !strings.Contains(a.Note, "5") {
+		t.Errorf("answer = %d rows, cut %v, note %q", len(a.Builds), a.Cut, a.Note)
+	}
+	for i := 1; i < len(a.Builds); i++ {
+		if a.Builds[i-1].Revision < a.Builds[i].Revision {
+			t.Errorf("not newest first: %d before %d", a.Builds[i-1].Revision, a.Builds[i].Revision)
+		}
+	}
+	if a.Builds[0].Revision != 1908 {
+		t.Errorf("newest revision = %d, want 1908", a.Builds[0].Revision)
+	}
+}
+
+// 1827/AC2: with no module the answer reads at most the limit's worth in key order (default 50), never the 2,000.
+func TestBuilderBuildsWithoutAModuleIsBounded(t *testing.T) {
+	b := newBig(2000)
+	out, err := run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	a := out.(view.BuildsAnswer)
+	if len(a.Builds) != view.DefaultBuildsLimit || !a.Cut || *b.reads > view.DefaultBuildsLimit {
+		t.Errorf("%d rows, cut %v, %d reads", len(a.Builds), a.Cut, *b.reads)
+	}
+	b = newBig(2000)
+	out, _ = run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{"state": "failed"})
+	if a := out.(view.BuildsAnswer); *b.reads != view.MaxBuildReads || !a.Cut || len(a.Builds) != 0 {
+		t.Errorf("a state matching nothing: %d reads (want the bound %d), cut %v", *b.reads, view.MaxBuildReads, a.Cut)
+	}
+}
+
+// 1818: fp2 together with commit is refused, naming both.
+func TestBuilderBuildRefusesFP2WithCommit(t *testing.T) {
+	r := both(fake{keyA: doneBuild}, fake{"netcheck": `{"v":3,"fp2":"` + fp2A + `","commit":"c1","path":"."}`})
+	_, err := run(t, r, view.BuilderBuildReadTool, map[string]any{"module": "netcheck", "fp2": fp2A, "commit": "c1"})
+	if err == nil || !strings.Contains(err.Error(), "fp2") || !strings.Contains(err.Error(), "commit") || !strings.Contains(err.Error(), "not both") {
+		t.Errorf("err = %v", err)
+	}
+}
+
+// 1681: indexer_candidates and indexer_candidate answer the stored value; a pull request's fp2 per touched module shows.
+func TestCandidateToolsAnswerTheStoredValue(t *testing.T) {
+	cand := `{"v":1,"group":"feat/x","base":3,"heads":"h","pulls":[{"modules":["zsh"],"fps":{"zsh":{"fp2":"` + fp2B + `","manifest":"m"}}}]}`
+	other := `{"v":1,"group":"fix","base":3,"heads":"h","pulls":[{"modules":["netcheck"]}]}`
+	r := fake{"feat/x": cand, "fix": other, "bad": "{"}.reader()
+	r.CandidatesState = r.State
+	out, err := run(t, r, view.CandidateReadTool, map[string]any{"group": "feat/x"})
+	if err != nil || string(out.(json.RawMessage)) != cand || !strings.Contains(string(out.(json.RawMessage)), fp2B) {
+		t.Fatalf("candidate = %v, %v", out, err)
+	}
+	if _, err := run(t, r, view.CandidateReadTool, map[string]any{"group": "gone"}); err == nil || !strings.Contains(err.Error(), "no candidate gone") {
+		t.Errorf("err = %v", err)
+	}
+	out, err = run(t, r, view.CandidatesReadTool, map[string]any{})
+	m := out.(map[string]any)
+	if err != nil || len(m["candidates"].(map[string]json.RawMessage)) != 2 || len(m["unreadable"].([]string)) != 1 {
+		t.Errorf("all = %v, %v", m, err)
+	}
+	out, _ = run(t, r, view.CandidatesReadTool, map[string]any{"module": "zsh"})
+	if c := out.(map[string]any)["candidates"].(map[string]json.RawMessage); len(c) != 1 || c["feat/x"] == nil {
+		t.Errorf("by module = %v", c)
+	}
+}

modules/mesh-delivery-view/internal/view/read.go

@@ -7,6 +7,7 @@ package view
 import (
 	"encoding/json"
 	"fmt"
+	"slices"
 	"sort"
 	"strings"
 
@@ -23,12 +24,16 @@ const (
 type StateRead struct {
 	Keys func() ([]string, error)
 	Get  func(key string) (json.RawMessage, error)
+	// GetEntry, when set, also gives the revision of the value; builder_builds orders by it.
+	GetEntry func(key string) (json.RawMessage, uint64, error)
 }
 
 // Reader answers the read tools from a modules state.
 type Reader struct {
 	State  StateRead // the indexer's modules
 	Builds StateRead // the builder's builds
+
+	CandidatesState StateRead // the indexer's candidates
 }
 
 // Module is the stored value of modules.<module>, exactly as the indexer wrote it.
@@ -84,6 +89,67 @@ func (r Reader) Modules(none2 bool) (map[string]json.RawMessage, error) {
 	return out, nil
 }
 
+// The read tools of the indexer's candidates (issue 1681): what the indexer stored under candidates.<group>, as stored, so the
+// new-form fingerprint of each touched module of an open pull request (pulls[].fps) can be read live.
+const (
+	CandidateReadTool  = "indexer_candidate"
+	CandidatesReadTool = "indexer_candidates"
+)
+
+// Candidate is the stored value of the candidates key of a group, exactly as the indexer wrote it.
+func (r Reader) Candidate(group string) (json.RawMessage, error) {
+	key, err := delivery.CandidatesKey(strings.TrimSpace(group))
+	if err != nil {
+		return nil, err
+	}
+	if r.CandidatesState.Get == nil {
+		return nil, fmt.Errorf("the candidates state cannot be read here")
+	}
+	raw, err := r.CandidatesState.Get(key)
+	if err != nil {
+		return nil, err
+	}
+	if len(raw) == 0 {
+		return nil, fmt.Errorf("no candidate %s in the candidates state", group)
+	}
+	return raw, nil
+}
+
+// Candidates is every candidate, by group, each as stored; with a module only the candidates with a pull request touching it.
+// A value that is not readable is listed under unreadable with its key, not skipped.
+func (r Reader) Candidates(module string) (map[string]json.RawMessage, []string, error) {
+	if r.CandidatesState.Keys == nil || r.CandidatesState.Get == nil {
+		return nil, nil, fmt.Errorf("the candidates state cannot be read here")
+	}
+	keys, err := r.CandidatesState.Keys()
+	if err != nil {
+		return nil, nil, err
+	}
+	sort.Strings(keys)
+	out := map[string]json.RawMessage{}
+	var unreadable []string
+	for _, k := range keys {
+		raw, err := r.CandidatesState.Get(k)
+		if err != nil {
+			return nil, nil, err
+		}
+		if len(raw) == 0 {
+			continue
+		}
+		group, perr := delivery.ParseCandidatesKey(k)
+		var c delivery.Candidate
+		if perr != nil || json.Unmarshal(raw, &c) != nil {
+			unreadable = append(unreadable, k)
+			continue
+		}
+		if module != "" && !slices.ContainsFunc(c.Pulls, func(p delivery.CandidatePull) bool { return slices.Contains(p.Modules, module) }) {
+			continue
+		}
+		out[group] = raw
+	}
+	return out, unreadable, nil
+}
+
 // The read tools of the builder's builds (novox/hq issue 1723): what the builder stored under `<module>.<fingerprint hex>`
 // in mesh-delivery-builder_builds, answered as stored, so its states, nodes (in the step) and digests can be checked live.
 const (
@@ -91,48 +157,100 @@ const (
 	BuilderBuildsReadTool = "builder_builds"
 )
 
-// BuilderRow is one key of the builder's builds: the key, the module and the fingerprint it parses to, and the stored value
-// as the builder wrote it (state, by, step, manifest, artifacts with digests, toolchains, error).
+// BuilderRow is one key of the builder's builds: the key, the module and the fingerprint it parses to, the revision of its
+// stored value (when the read gives one) and the stored value as the builder wrote it.
 type BuilderRow struct {
 	Key         string          `json:"key"`
 	Module      string          `json:"module"`
 	Fingerprint string          `json:"fingerprint"`
+	Revision    uint64          `json:"revision,omitempty"`
 	Build       json.RawMessage `json:"build"`
 }
 
+// The bounds of builder_builds (issue 1827): the rows it returns when no limit is given, and the values it reads in one call.
+// Reading a value is a bus round trip, so the answer must not read the whole bucket (it timed out at 30 s on the live one).
+const (
+	DefaultBuildsLimit = 50
+	MaxBuildReads      = 200
+)
+
+// BuildsAnswer is the answer of builder_builds: the rows, newest revision first, whether the answer was cut and why, and
+// the keys that are no build key (Unreadable), which are listed rather than silently skipped (issue 1818).
+type BuildsAnswer struct {
+	Builds     []BuilderRow `json:"builds"`
+	Cut        bool         `json:"cut,omitempty"`
+	Note       string       `json:"note,omitempty"`
+	Unreadable []string     `json:"unreadable,omitempty"`
+}
+
+func (r Reader) get(key string) (json.RawMessage, uint64, error) {
+	if r.Builds.GetEntry != nil {
+		return r.Builds.GetEntry(key)
+	}
+	raw, err := r.Builds.Get(key)
+	return raw, 0, err
+}
+
 func (r Reader) builderRow(key string) (BuilderRow, bool, error) {
 	module, fp, err := delivery.ParseBuilderBuildKey(key)
 	if err != nil {
 		return BuilderRow{}, false, nil // a key that is no build key is not a build
 	}
-	raw, err := r.Builds.Get(key)
+	raw, rev, err := r.get(key)
 	if err != nil {
 		return BuilderRow{}, false, err
 	}
 	if len(raw) == 0 {
 		return BuilderRow{}, false, nil
 	}
-	return BuilderRow{Key: key, Module: module, Fingerprint: fp, Build: raw}, true, nil
+	return BuilderRow{Key: key, Module: module, Fingerprint: fp, Revision: rev, Build: raw}, true, nil
 }
 
-// BuilderBuilds lists the builder's keys, each as stored, sorted by key; module and state, when given, keep only the builds
-// of that module or in that state.
-func (r Reader) BuilderBuilds(module, state string) ([]BuilderRow, error) {
+// BuilderBuilds lists the builder's keys, each as stored, newest revision first. With a module it considers only the keys
+// that begin `<module>.`, before any value is read; with a state it keeps the builds in that state. It returns at most limit
+// rows (DefaultBuildsLimit when limit is not above zero) and reads at most MaxBuildReads values, and says when it cut the
+// answer. Keys that are no build key are listed as unreadable. Without a module the keys are walked in key order, so the
+// answer is the first rows found, not the newest of the bucket: give a module to get its newest.
+func (r Reader) BuilderBuilds(module, state string, limit int) (BuildsAnswer, error) {
 	if r.Builds.Keys == nil || r.Builds.Get == nil {
-		return nil, fmt.Errorf("the builder's builds cannot be read here")
+		return BuildsAnswer{}, fmt.Errorf("the builder's builds cannot be read here")
+	}
+	if limit <= 0 {
+		limit = DefaultBuildsLimit
 	}
 	keys, err := r.Builds.Keys()
 	if err != nil {
-		return nil, err
+		return BuildsAnswer{}, err
 	}
 	sort.Strings(keys)
-	rows := []BuilderRow{}
+	prefix := ""
+	if module = strings.TrimSpace(module); module != "" {
+		prefix = module + "."
+	}
+	out := BuildsAnswer{Builds: []BuilderRow{}}
+	var candidates []string
 	for _, k := range keys {
+		if prefix != "" && !strings.HasPrefix(k, prefix) {
+			continue
+		}
+		if _, _, err := delivery.ParseBuilderBuildKey(k); err != nil {
+			out.Unreadable = append(out.Unreadable, k)
+			continue
+		}
+		candidates = append(candidates, k)
+	}
+	reads := 0
+	for _, k := range candidates {
+		if reads >= MaxBuildReads {
+			out.Cut, out.Note = true, fmt.Sprintf("read %d values and stopped; give a module (and a state) to narrow the answer", MaxBuildReads)
+			break
+		}
+		reads++
 		row, ok, err := r.builderRow(k)
 		if err != nil {
-			return nil, err
+			return BuildsAnswer{}, err
 		}
-		if !ok || (module != "" && row.Module != module) {
+		if !ok {
 			continue
 		}
 		if state != "" {
@@ -143,9 +261,26 @@ func (r Reader) BuilderBuilds(module, state string) ([]BuilderRow, error) {
 				continue
 			}
 		}
-		rows = append(rows, row)
+		out.Builds = append(out.Builds, row)
+		// Without a module the walk is in key order and stops at the limit; with one, all its (few) keys are read to order them.
+		if prefix == "" && len(out.Builds) >= limit {
+			if reads < len(candidates) {
+				out.Cut, out.Note = true, fmt.Sprintf("stopped at the limit of %d builds in key order, not newest first; give a module for its newest", limit)
+			}
+			break
+		}
 	}
-	return rows, nil
+	sort.SliceStable(out.Builds, func(i, j int) bool {
+		if out.Builds[i].Revision != out.Builds[j].Revision {
+			return out.Builds[i].Revision > out.Builds[j].Revision
+		}
+		return out.Builds[i].Key < out.Builds[j].Key
+	})
+	if len(out.Builds) > limit {
+		out.Builds = out.Builds[:limit]
+		out.Cut, out.Note = true, fmt.Sprintf("cut to the newest %d builds of %s", limit, module)
+	}
+	return out, nil
 }
 
 // BuilderBuild is the build of module at the new-form fingerprint fp2; with an empty fp2 it is the build at the fingerprint
@@ -156,6 +291,9 @@ func (r Reader) BuilderBuild(module, fp2, commit string) (BuilderRow, error) {
 	if r.Builds.Get == nil {
 		return BuilderRow{}, fmt.Errorf("the builder's builds cannot be read here")
 	}
+	if fp2 != "" && commit != "" {
+		return BuilderRow{}, fmt.Errorf("give fp2 or commit, not both: fp2 %s names a build directly, commit %s asks for the build at the indexer's fingerprint of that commit", fp2, commit)
+	}
 	if fp2 == "" {
 		raw, err := r.Module(module)
 		if err != nil {

modules/mesh-delivery-view/manifest_test.go

@@ -95,7 +95,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
 	}
 	// The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer
 	// itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads.
-	if want := []string{"indexer_module", "indexer_modules", "builder_build", "builder_builds"}; !reflect.DeepEqual(m.Tools, want) {
+	if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds"}; !reflect.DeepEqual(m.Tools, want) {
 		t.Errorf("tools = %v, want %v", m.Tools, want)
 	}
 	for _, a := range m.Build.Artifacts {

modules/mesh-delivery-view/module.json

@@ -6,6 +6,8 @@
   "tools": [
     "indexer_module",
     "indexer_modules",
+    "indexer_candidate",
+    "indexer_candidates",
     "builder_build",
     "builder_builds"
   ],