2 Commits
11 changed files with 575 additions and 29 deletions
@@ -59,11 +59,52 @@ func Tools(r check.Reader) []stdio.Tool {
kept, err := r.Kept(level, digest)
return map[string]any{"kept": kept}, err
}},
{Name: check.PreviewsReadTool,
Description: "The merge previews the checker holds, each as stored under previews.<repository>.<head>.<trunk>: the merge commit, per module directory " +
"its tree, manifest digest and lock hashes, or the files in conflict. With a repository, that repository's; with a head (7 or more characters) " +
"only that head's. Keys that are no preview key are listed as unreadable. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
"head": map[string]any{"type": "string", "description": "a pull request head commit, or a prefix of at least 7 characters"}}},
Run: func(args map[string]any) (any, error) {
repository, _ := args["repository"].(string)
head, _ := args["head"].(string)
if head != "" && len(head) < 7 {
return nil, fmt.Errorf("the head %q is too short: give at least 7 characters", head)
}
ps, bad, err := r.Previews(repository, head)
return map[string]any{"previews": ps, "unreadable": bad}, err
}},
{Name: check.StatusReadTool,
Description: "What holds the checker back and what runs: the capacity per node (the setting, the default one step per node), whether the controller's " +
"delivery plans in their build phase hold a key (no new check starts while one does), the steps in flight on the build modules by node, and the " +
"candidates without a verdict for their heads. The steps the checker holds waiting for a place are in its process, not in a bucket, and the " +
"answer says so. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return r.Status() }},
}
}
func capacity() (string, error) {
path := os.Getenv("MESH_CHECKER_SETTINGS")
if path == "" {
return os.Getenv("MESH_CHECKER_CAPACITY"), nil
}
raw, err := os.ReadFile(path)
if err != nil {
return "", err
}
var s struct {
Capacity string `json:"capacity"`
}
return s.Capacity, json.Unmarshal(raw, &s)
}
func main() {
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates"))}
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")),
PreviewState: read(stdio.State("previews")), PlansState: read(stdio.State("mesh-controller.plans-building")),
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
Capacity: capacity}
if err := stdio.Serve("", Tools(r)); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
os.Exit(1)
@@ -1,12 +1,13 @@
package main
import (
"slices"
"testing"
"mesh-delivery-checker/internal/check"
)
func TestTheToolsServedAreTheThreeReadTools(t *testing.T) {
func TestTheToolsServedAreTheFiveReadTools(t *testing.T) {
var names []string
for _, tl := range Tools(check.Reader{}) {
names = append(names, tl.Name)
@@ -14,12 +15,15 @@ func TestTheToolsServedAreTheThreeReadTools(t *testing.T) {
t.Errorf("%s has no description or no run", tl.Name)
}
}
want := []string{check.VerdictsReadTool, check.PullReadTool, check.KeptReadTool}
if len(names) != 3 || names[0] != want[0] || names[1] != want[1] || names[2] != want[2] {
want := []string{check.VerdictsReadTool, check.PullReadTool, check.KeptReadTool, check.PreviewsReadTool, check.StatusReadTool}
if !slices.Equal(names, want) {
t.Errorf("tools = %v, want %v", names, want)
}
// With nothing to read from a tool says so and changes nothing.
for _, tl := range Tools(check.Reader{}) {
if tl.Name == check.StatusReadTool {
continue // it answers from what it can read, and says what it could not
}
if _, err := tl.Run(map[string]any{"repository": "novox/x", "number": float64(1)}); err == nil {
t.Errorf("%s answered with no state to read", tl.Name)
}
@@ -7,6 +7,7 @@ import (
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the
@@ -17,6 +18,9 @@ const (
VerdictsReadTool = "checker_verdicts"
PullReadTool = "checker_pull"
KeptReadTool = "checker_kept"
// PreviewsReadTool reads the previews state (issue 1680); StatusReadTool reads what the checker is held by and what it runs (issue 1685).
PreviewsReadTool = "checker_previews"
StatusReadTool = "checker_status"
)
// StateRead reads one state: its keys and a key's stored value (nil for a key with none).
@@ -26,7 +30,14 @@ type StateRead struct {
}
// Reader answers the read tools from the checker's verdicts and kept states and the indexer's candidates.
type Reader struct{ VerdictState, KeptState, CandidateState StateRead }
type Reader struct {
VerdictState, KeptState, CandidateState StateRead
PreviewState StateRead // the checker's merge previews
PlansState StateRead // the controller's delivery plans in their build phase; zero where the bucket is absent
StepStates map[string]StateRead // the build modules' steps states, by module
Capacity func() (string, error) // the capacity setting as written, `node=steps,...`
}
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
// candidate still holds those heads; without it the report is that of the verdict alone.
@@ -191,3 +202,153 @@ func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, error) {
}
return out, nil
}
// PreviewRead is one preview as stored, with the parts of its key.
type PreviewRead struct {
Key string `json:"key"`
Repository string `json:"repository"`
Head string `json:"head"`
Trunk string `json:"trunk"`
Preview json.RawMessage `json:"preview"`
}
// Previews are the previews the checker holds, each as stored. With a repository only that repository's; with a head (a
// prefix of at least 7 characters is enough) only previews of that head. A key that is no preview key is listed as unreadable.
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, error) {
if r.PreviewState.Keys == nil || r.PreviewState.Get == nil {
return nil, nil, fmt.Errorf("the previews state cannot be read here")
}
keys, err := r.PreviewState.Keys()
if err != nil {
return nil, nil, err
}
sort.Strings(keys)
out := []PreviewRead{}
var bad []string
for _, k := range keys {
repo, h, trunk, err := delivery.ParsePreviewKey(k)
if err != nil {
bad = append(bad, k)
continue
}
if (repository != "" && repo != repository) || (head != "" && !strings.HasPrefix(h, head)) {
continue
}
raw, err := r.PreviewState.Get(k)
if err != nil {
return nil, nil, err
}
if len(raw) > 0 {
out = append(out, PreviewRead{Key: k, Repository: repo, Head: h, Trunk: trunk, Preview: raw})
}
}
return out, bad, nil
}
// RunningStep is a step in flight on a build module.
type RunningStep struct {
Module string `json:"module"`
Node string `json:"node"`
ID string `json:"id"`
Tool string `json:"tool"`
Since string `json:"since,omitempty"`
}
// Status is what holds the checker back and what runs now, read from the buckets: the checker's held and running steps live in
// its process, so they are shown as what the buckets say (the steps in flight on the build modules, the candidates without a
// verdict for their heads, and the delivery plans in their build phase), not as the process's own counters.
type Status struct {
Capacity map[string]int `json:"capacity"`
CapacityNote string `json:"capacity_note,omitempty"`
DefaultPerNode int `json:"default_per_node"`
PlansBuilding struct {
Readable bool `json:"readable"`
Keys []string `json:"keys"`
Holds bool `json:"holds"`
} `json:"plans_building"`
Running []RunningStep `json:"running_steps"`
RunningByNode map[string]int `json:"running_by_node"`
Unjudged []string `json:"unjudged_candidates"`
Notes []string `json:"notes,omitempty"`
}
// Status reads the capacity, the plan bucket, the build modules' steps and the candidates against their verdicts.
func (r Reader) Status() (Status, error) {
st := Status{DefaultPerNode: buildrun.DefaultCapacity, Capacity: map[string]int{}, RunningByNode: map[string]int{}}
if r.Capacity == nil {
st.CapacityNote = "the capacity setting cannot be read from here"
} else if setting, err := r.Capacity(); err != nil {
st.CapacityNote = "the capacity setting cannot be read: " + err.Error()
} else if st.Capacity, err = buildrun.ParseCapacity(setting); err != nil {
st.Capacity, st.CapacityNote = map[string]int{}, "the capacity setting is not valid: "+err.Error()
}
if r.PlansState.Keys == nil {
st.Notes = append(st.Notes, PlansBuildingBucket+" is not readable here (absent), so it is read as empty: no delivery holds the checker")
} else if keys, err := r.PlansState.Keys(); err != nil {
st.Notes = append(st.Notes, PlansBuildingBucket+" cannot be listed, so it is read as empty: "+err.Error())
} else {
sort.Strings(keys)
st.PlansBuilding.Readable, st.PlansBuilding.Keys, st.PlansBuilding.Holds = true, keys, len(keys) > 0
}
mods := make([]string, 0, len(r.StepStates))
for m := range r.StepStates {
mods = append(mods, m)
}
sort.Strings(mods)
st.Running = []RunningStep{}
for _, m := range mods {
sr := r.StepStates[m]
if sr.Keys == nil || sr.Get == nil {
continue
}
keys, err := sr.Keys()
if err != nil {
st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error())
continue
}
sort.Strings(keys)
for _, k := range keys {
raw, err := sr.Get(k)
if err != nil || len(raw) == 0 {
continue
}
step, err := delivery.DecodeStep(raw)
if err != nil || step.State != delivery.StepRunning {
continue
}
rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool}
if !step.Started.IsZero() {
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
}
st.Running = append(st.Running, rs)
st.RunningByNode[step.Node]++
}
}
st.Unjudged = []string{}
if r.CandidateState.Keys != nil && r.CandidateState.Get != nil && r.VerdictState.Get != nil {
keys, err := r.CandidateState.Keys()
if err != nil {
return st, err
}
sort.Strings(keys)
for _, k := range keys {
raw, err := r.CandidateState.Get(k)
if err != nil || len(raw) == 0 {
continue
}
c, err := delivery.DecodeCandidate(raw)
if err != nil {
continue
}
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
if err != nil {
continue
}
if v, err := r.VerdictState.Get(vkey); err == nil && len(v) == 0 {
st.Unjudged = append(st.Unjudged, c.Group)
}
}
}
st.Notes = append(st.Notes, "the steps the checker holds waiting for a place are in its process and in no bucket: a candidate without a verdict while a plan key exists, or while its node runs as many steps as its capacity, is held")
return st, nil
}
@@ -3,8 +3,10 @@ package check
import (
"context"
"encoding/json"
"os"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -102,3 +104,64 @@ func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { //
t.Error("a digest nothing kept was answered")
}
}
// 1680 live acceptance: checker_previews shows the preview key of an open pull request, as stored, narrowed by repository and head.
func TestThePreviewsToolShowsTheStoredPreviewOfAPullRequest(t *testing.T) {
pv := delivery.Preview{Repository: repoA, Head: head1, Trunk: head2, Merge: "m1", Modules: map[string]delivery.PreviewModule{}}
data, err := delivery.EncodePreview(pv)
if err != nil {
t.Fatal(err)
}
key, _ := delivery.PreviewKey(repoA, head1, head2)
rd := Reader{PreviewState: fakeState{key: data, "junk": []byte("{}")}.read()}
got, bad, err := rd.Previews(repoA, head1[:8])
if err != nil || len(got) != 1 || got[0].Key != key || got[0].Head != head1 || got[0].Trunk != head2 || string(got[0].Preview) != string(data) {
t.Fatalf("previews = %+v, %v", got, err)
}
if len(bad) != 1 || bad[0] != "junk" {
t.Errorf("unreadable = %v", bad)
}
if got, _, _ := rd.Previews("novox/other", ""); len(got) != 0 {
t.Errorf("another repository's = %+v", got)
}
if _, _, err := (Reader{}).Previews("", ""); err == nil {
t.Error("answered with no state to read")
}
}
// 1685 live acceptance: checker_status reads the capacity, whether a plan key holds the checker, the steps in flight by node and
// the candidates without a verdict.
func TestTheStatusToolReadsCapacityHoldsAndRunningSteps(t *testing.T) {
running, _ := delivery.EncodeStep(delivery.Step{ID: "abc1", Tool: "build_go_test", Node: "shanks", State: delivery.StepRunning, Started: time.Date(2026, 10, 22, 15, 0, 0, 0, time.UTC)})
passed, _ := delivery.EncodeStep(delivery.Step{ID: "abc2", Tool: "build_go_test", Node: "ace", State: delivery.StepPassed})
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
ckey, _ := delivery.CandidatesKey("feat/x")
rd := Reader{
Capacity: func() (string, error) { return "shanks=2,ace=0", nil },
PlansState: fakeState{"p1": []byte("{}")}.read(),
StepStates: map[string]StateRead{"mesh-build-go": fakeState{"shanks.abc1": running, "ace.abc2": passed}.read()},
CandidateState: fakeState{ckey: cand}.read(),
VerdictState: fakeState{}.read(),
}
st, err := rd.Status()
if err != nil {
t.Fatal(err)
}
if st.Capacity["shanks"] != 2 || st.Capacity["ace"] != 0 || st.DefaultPerNode != 1 {
t.Errorf("capacity = %v default %d", st.Capacity, st.DefaultPerNode)
}
if !st.PlansBuilding.Holds || len(st.PlansBuilding.Keys) != 1 {
t.Errorf("plans = %+v", st.PlansBuilding)
}
if len(st.Running) != 1 || st.Running[0].Node != "shanks" || st.RunningByNode["shanks"] != 1 || st.RunningByNode["ace"] != 0 {
t.Errorf("running = %+v %v", st.Running, st.RunningByNode)
}
if len(st.Unjudged) != 1 || st.Unjudged[0] != "feat/x" {
t.Errorf("unjudged = %v", st.Unjudged)
}
// An absent plan bucket is read as empty and said; an unreadable capacity is said, not guessed.
st, err = Reader{Capacity: func() (string, error) { return "", os.ErrNotExist }}.Status()
if err != nil || st.PlansBuilding.Holds || st.CapacityNote == "" || len(st.Notes) == 0 {
t.Errorf("empty status = %+v, %v", st, err)
}
}
@@ -118,7 +118,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
t.Errorf("streams = %+v, want %+v", m.Streams, want)
}
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
if want := []string{"checker_kept", "checker_pull", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats", "bus"} {
@@ -31,7 +31,9 @@
],
"tools": [
"checker_kept",
"checker_previews",
"checker_pull",
"checker_status",
"checker_verdicts"
],
"invokes": [
@@ -35,18 +35,39 @@ func Tools(r view.Reader) []stdio.Tool {
mods, err := r.Modules(only)
return map[string]any{"modules": mods}, err
}},
{Name: view.CandidateReadTool,
Description: "What the indexer stored for one candidate (the open pull requests of a branch) under candidates.<group>, as stored: " +
"the pull requests and heads, the base, and per pull request the touched modules with their new-form fingerprint (fp2) or why " +
"they have none (none2). Read-only.",
Input: map[string]any{"type": "object", "required": []string{"group"}, "properties": map[string]any{
"group": map[string]any{"type": "string", "description": "the candidate's group, the branch name its pull requests share"}}},
Run: func(args map[string]any) (any, error) {
group, _ := args["group"].(string)
return r.Candidate(group)
}},
{Name: view.CandidatesReadTool,
Description: "Every candidate the indexer holds, by group, each as stored under candidates.<group>; with module, only those with a " +
"pull request touching that module. Keys that cannot be read are listed as unreadable. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "only candidates touching this module"}}},
Run: func(args map[string]any) (any, error) {
module, _ := args["module"].(string)
c, bad, err := r.Candidates(module)
return map[string]any{"candidates": c, "unreadable": bad}, err
}},
{Name: view.BuilderBuildsReadTool,
Description: "The builder's build keys, each as stored in mesh-delivery-builder_builds: key, module, fingerprint (src2) and build " +
"(state, by, step with the nodes it was built on, manifest, artifacts with their digests, toolchains, error). Narrow by module " +
"or state (queued, building, done, failed). Read-only.",
"or state (queued, building, done, failed); at most limit builds (default 50), newest revision first for a module, and the answer says when it was cut and lists keys that are no build key (unreadable). Without a module, give one to get its newest. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "only this module's builds"},
"state": map[string]any{"type": "string", "description": "only builds in this state"}}},
"state": map[string]any{"type": "string", "description": "only builds in this state"},
"limit": map[string]any{"type": "integer", "description": "at most this many builds (default 50), newest revision first"}}},
Run: func(args map[string]any) (any, error) {
module, _ := args["module"].(string)
state, _ := args["state"].(string)
rows, err := r.BuilderBuilds(module, state)
return map[string]any{"builds": rows}, err
limit, _ := args["limit"].(float64)
return r.BuilderBuilds(module, state, int(limit))
}},
{Name: view.BuilderBuildReadTool,
Description: "One build of the builder as stored. Give the module and its fingerprint (fp2, src2:...), or the module alone for the build " +
@@ -65,7 +86,13 @@ func Tools(r view.Reader) []stdio.Tool {
}
func read(state stdio.KeptState) view.StateRead {
return view.StateRead{Keys: state.Keys, Get: func(key string) (json.RawMessage, error) {
return view.StateRead{Keys: state.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) {
e, err := state.Get(key)
if err != nil || e == nil {
return nil, 0, err
}
return e.Value, e.Revision, nil
}, Get: func(key string) (json.RawMessage, error) {
e, err := state.Get(key)
if err != nil || e == nil {
return nil, err
@@ -75,7 +102,7 @@ func read(state stdio.KeptState) view.StateRead {
}
func main() {
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds"))}
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates"))}
if err := stdio.Serve("", Tools(r)); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err)
os.Exit(1)
@@ -2,6 +2,7 @@ package main
import (
"encoding/json"
"fmt"
"strings"
"testing"
@@ -117,13 +118,16 @@ func TestBuilderBuildsListsKeysAsStored(t *testing.T) {
if err != nil {
t.Fatal(err)
}
rows := out.(map[string]any)["builds"].([]view.BuilderRow)
rows := out.(view.BuildsAnswer).Builds
if u := out.(view.BuildsAnswer).Unreadable; len(u) != 1 || u[0] != "junk" {
t.Errorf("unreadable = %v, want [junk] (issue 1818)", u)
}
if len(rows) != 2 || rows[0].Key != keyA || rows[0].Module != "netcheck" || rows[0].Fingerprint != fp2A || string(rows[0].Build) != doneBuild || string(rows[1].Build) != failBuild {
t.Errorf("rows = %+v", rows)
}
for args, want := range map[string]string{"state": keyB, "module": keyA} {
out, _ := run(t, r, view.BuilderBuildsReadTool, map[string]any{args: map[string]string{keyB: "failed", keyA: "netcheck"}[want]})
got := out.(map[string]any)["builds"].([]view.BuilderRow)
got := out.(view.BuildsAnswer).Builds
if len(got) != 1 || got[0].Key != want {
t.Errorf("%s filter = %+v, want %s", args, got, want)
}
@@ -156,3 +160,107 @@ func TestBuilderBuildJoinsTheIndexersFingerprint(t *testing.T) {
t.Errorf("an unbuilt fingerprint: %v", err)
}
}
// bigBuilds is a bucket of n keys over 100 modules that counts the values read.
type bigBuilds struct {
fake fake
reads *int
}
func newBig(n int) bigBuilds {
f := fake{}
for i := 0; i < n; i++ {
f[fmt.Sprintf("mod%02d.%064x", i%100, i)] = doneBuild
}
return bigBuilds{fake: f, reads: new(int)}
}
func (b bigBuilds) reader() view.Reader {
r := b.fake.reader()
get := r.State.Get
r.Builds = view.StateRead{Keys: r.State.Keys, Get: func(k string) (json.RawMessage, error) { *b.reads++; return get(k) },
GetEntry: func(k string) (json.RawMessage, uint64, error) {
*b.reads++
raw, err := get(k)
var rev uint64
fmt.Sscanf(k[strings.Index(k, ".")+1:], "%x", &rev) // the fake's revision is the key's number
return raw, rev + 1, err
}}
return r
}
// 1827/AC1: with 2,000 keys, a module's builds are found by key prefix, so only that module's values are read (20, not 2,000),
// newest revision first, within the limit, and the answer says it was cut.
func TestBuilderBuildsFiltersByPrefixBeforeAnyGet(t *testing.T) {
b := newBig(2000)
out, err := run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{"module": "mod07", "limit": float64(5)})
if err != nil {
t.Fatal(err)
}
a := out.(view.BuildsAnswer)
if *b.reads != 20 {
t.Errorf("read %d values for a module of 20 keys, want 20", *b.reads)
}
if len(a.Builds) != 5 || !a.Cut || !strings.Contains(a.Note, "5") {
t.Errorf("answer = %d rows, cut %v, note %q", len(a.Builds), a.Cut, a.Note)
}
for i := 1; i < len(a.Builds); i++ {
if a.Builds[i-1].Revision < a.Builds[i].Revision {
t.Errorf("not newest first: %d before %d", a.Builds[i-1].Revision, a.Builds[i].Revision)
}
}
if a.Builds[0].Revision != 1908 {
t.Errorf("newest revision = %d, want 1908", a.Builds[0].Revision)
}
}
// 1827/AC2: with no module the answer reads at most the limit's worth in key order (default 50), never the 2,000.
func TestBuilderBuildsWithoutAModuleIsBounded(t *testing.T) {
b := newBig(2000)
out, err := run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{})
if err != nil {
t.Fatal(err)
}
a := out.(view.BuildsAnswer)
if len(a.Builds) != view.DefaultBuildsLimit || !a.Cut || *b.reads > view.DefaultBuildsLimit {
t.Errorf("%d rows, cut %v, %d reads", len(a.Builds), a.Cut, *b.reads)
}
b = newBig(2000)
out, _ = run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{"state": "failed"})
if a := out.(view.BuildsAnswer); *b.reads != view.MaxBuildReads || !a.Cut || len(a.Builds) != 0 {
t.Errorf("a state matching nothing: %d reads (want the bound %d), cut %v", *b.reads, view.MaxBuildReads, a.Cut)
}
}
// 1818: fp2 together with commit is refused, naming both.
func TestBuilderBuildRefusesFP2WithCommit(t *testing.T) {
r := both(fake{keyA: doneBuild}, fake{"netcheck": `{"v":3,"fp2":"` + fp2A + `","commit":"c1","path":"."}`})
_, err := run(t, r, view.BuilderBuildReadTool, map[string]any{"module": "netcheck", "fp2": fp2A, "commit": "c1"})
if err == nil || !strings.Contains(err.Error(), "fp2") || !strings.Contains(err.Error(), "commit") || !strings.Contains(err.Error(), "not both") {
t.Errorf("err = %v", err)
}
}
// 1681: indexer_candidates and indexer_candidate answer the stored value; a pull request's fp2 per touched module shows.
func TestCandidateToolsAnswerTheStoredValue(t *testing.T) {
cand := `{"v":1,"group":"feat/x","base":3,"heads":"h","pulls":[{"modules":["zsh"],"fps":{"zsh":{"fp2":"` + fp2B + `","manifest":"m"}}}]}`
other := `{"v":1,"group":"fix","base":3,"heads":"h","pulls":[{"modules":["netcheck"]}]}`
r := fake{"feat/x": cand, "fix": other, "bad": "{"}.reader()
r.CandidatesState = r.State
out, err := run(t, r, view.CandidateReadTool, map[string]any{"group": "feat/x"})
if err != nil || string(out.(json.RawMessage)) != cand || !strings.Contains(string(out.(json.RawMessage)), fp2B) {
t.Fatalf("candidate = %v, %v", out, err)
}
if _, err := run(t, r, view.CandidateReadTool, map[string]any{"group": "gone"}); err == nil || !strings.Contains(err.Error(), "no candidate gone") {
t.Errorf("err = %v", err)
}
out, err = run(t, r, view.CandidatesReadTool, map[string]any{})
m := out.(map[string]any)
if err != nil || len(m["candidates"].(map[string]json.RawMessage)) != 2 || len(m["unreadable"].([]string)) != 1 {
t.Errorf("all = %v, %v", m, err)
}
out, _ = run(t, r, view.CandidatesReadTool, map[string]any{"module": "zsh"})
if c := out.(map[string]any)["candidates"].(map[string]json.RawMessage); len(c) != 1 || c["feat/x"] == nil {
t.Errorf("by module = %v", c)
}
}
+152 -14
View File
@@ -7,6 +7,7 @@ package view
import (
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
@@ -23,12 +24,16 @@ const (
type StateRead struct {
Keys func() ([]string, error)
Get func(key string) (json.RawMessage, error)
// GetEntry, when set, also gives the revision of the value; builder_builds orders by it.
GetEntry func(key string) (json.RawMessage, uint64, error)
}
// Reader answers the read tools from a modules state.
type Reader struct {
State StateRead // the indexer's modules
Builds StateRead // the builder's builds
CandidatesState StateRead // the indexer's candidates
}
// Module is the stored value of modules.<module>, exactly as the indexer wrote it.
@@ -84,6 +89,67 @@ func (r Reader) Modules(none2 bool) (map[string]json.RawMessage, error) {
return out, nil
}
// The read tools of the indexer's candidates (issue 1681): what the indexer stored under candidates.<group>, as stored, so the
// new-form fingerprint of each touched module of an open pull request (pulls[].fps) can be read live.
const (
CandidateReadTool = "indexer_candidate"
CandidatesReadTool = "indexer_candidates"
)
// Candidate is the stored value of the candidates key of a group, exactly as the indexer wrote it.
func (r Reader) Candidate(group string) (json.RawMessage, error) {
key, err := delivery.CandidatesKey(strings.TrimSpace(group))
if err != nil {
return nil, err
}
if r.CandidatesState.Get == nil {
return nil, fmt.Errorf("the candidates state cannot be read here")
}
raw, err := r.CandidatesState.Get(key)
if err != nil {
return nil, err
}
if len(raw) == 0 {
return nil, fmt.Errorf("no candidate %s in the candidates state", group)
}
return raw, nil
}
// Candidates is every candidate, by group, each as stored; with a module only the candidates with a pull request touching it.
// A value that is not readable is listed under unreadable with its key, not skipped.
func (r Reader) Candidates(module string) (map[string]json.RawMessage, []string, error) {
if r.CandidatesState.Keys == nil || r.CandidatesState.Get == nil {
return nil, nil, fmt.Errorf("the candidates state cannot be read here")
}
keys, err := r.CandidatesState.Keys()
if err != nil {
return nil, nil, err
}
sort.Strings(keys)
out := map[string]json.RawMessage{}
var unreadable []string
for _, k := range keys {
raw, err := r.CandidatesState.Get(k)
if err != nil {
return nil, nil, err
}
if len(raw) == 0 {
continue
}
group, perr := delivery.ParseCandidatesKey(k)
var c delivery.Candidate
if perr != nil || json.Unmarshal(raw, &c) != nil {
unreadable = append(unreadable, k)
continue
}
if module != "" && !slices.ContainsFunc(c.Pulls, func(p delivery.CandidatePull) bool { return slices.Contains(p.Modules, module) }) {
continue
}
out[group] = raw
}
return out, unreadable, nil
}
// The read tools of the builder's builds (novox/hq issue 1723): what the builder stored under `<module>.<fingerprint hex>`
// in mesh-delivery-builder_builds, answered as stored, so its states, nodes (in the step) and digests can be checked live.
const (
@@ -91,48 +157,100 @@ const (
BuilderBuildsReadTool = "builder_builds"
)
// BuilderRow is one key of the builder's builds: the key, the module and the fingerprint it parses to, and the stored value
// as the builder wrote it (state, by, step, manifest, artifacts with digests, toolchains, error).
// BuilderRow is one key of the builder's builds: the key, the module and the fingerprint it parses to, the revision of its
// stored value (when the read gives one) and the stored value as the builder wrote it.
type BuilderRow struct {
Key string `json:"key"`
Module string `json:"module"`
Fingerprint string `json:"fingerprint"`
Revision uint64 `json:"revision,omitempty"`
Build json.RawMessage `json:"build"`
}
// The bounds of builder_builds (issue 1827): the rows it returns when no limit is given, and the values it reads in one call.
// Reading a value is a bus round trip, so the answer must not read the whole bucket (it timed out at 30 s on the live one).
const (
DefaultBuildsLimit = 50
MaxBuildReads = 200
)
// BuildsAnswer is the answer of builder_builds: the rows, newest revision first, whether the answer was cut and why, and
// the keys that are no build key (Unreadable), which are listed rather than silently skipped (issue 1818).
type BuildsAnswer struct {
Builds []BuilderRow `json:"builds"`
Cut bool `json:"cut,omitempty"`
Note string `json:"note,omitempty"`
Unreadable []string `json:"unreadable,omitempty"`
}
func (r Reader) get(key string) (json.RawMessage, uint64, error) {
if r.Builds.GetEntry != nil {
return r.Builds.GetEntry(key)
}
raw, err := r.Builds.Get(key)
return raw, 0, err
}
func (r Reader) builderRow(key string) (BuilderRow, bool, error) {
module, fp, err := delivery.ParseBuilderBuildKey(key)
if err != nil {
return BuilderRow{}, false, nil // a key that is no build key is not a build
}
raw, err := r.Builds.Get(key)
raw, rev, err := r.get(key)
if err != nil {
return BuilderRow{}, false, err
}
if len(raw) == 0 {
return BuilderRow{}, false, nil
}
return BuilderRow{Key: key, Module: module, Fingerprint: fp, Build: raw}, true, nil
return BuilderRow{Key: key, Module: module, Fingerprint: fp, Revision: rev, Build: raw}, true, nil
}
// BuilderBuilds lists the builder's keys, each as stored, sorted by key; module and state, when given, keep only the builds
// of that module or in that state.
func (r Reader) BuilderBuilds(module, state string) ([]BuilderRow, error) {
// BuilderBuilds lists the builder's keys, each as stored, newest revision first. With a module it considers only the keys
// that begin `<module>.`, before any value is read; with a state it keeps the builds in that state. It returns at most limit
// rows (DefaultBuildsLimit when limit is not above zero) and reads at most MaxBuildReads values, and says when it cut the
// answer. Keys that are no build key are listed as unreadable. Without a module the keys are walked in key order, so the
// answer is the first rows found, not the newest of the bucket: give a module to get its newest.
func (r Reader) BuilderBuilds(module, state string, limit int) (BuildsAnswer, error) {
if r.Builds.Keys == nil || r.Builds.Get == nil {
return nil, fmt.Errorf("the builder's builds cannot be read here")
return BuildsAnswer{}, fmt.Errorf("the builder's builds cannot be read here")
}
if limit <= 0 {
limit = DefaultBuildsLimit
}
keys, err := r.Builds.Keys()
if err != nil {
return nil, err
return BuildsAnswer{}, err
}
sort.Strings(keys)
rows := []BuilderRow{}
prefix := ""
if module = strings.TrimSpace(module); module != "" {
prefix = module + "."
}
out := BuildsAnswer{Builds: []BuilderRow{}}
var candidates []string
for _, k := range keys {
if prefix != "" && !strings.HasPrefix(k, prefix) {
continue
}
if _, _, err := delivery.ParseBuilderBuildKey(k); err != nil {
out.Unreadable = append(out.Unreadable, k)
continue
}
candidates = append(candidates, k)
}
reads := 0
for _, k := range candidates {
if reads >= MaxBuildReads {
out.Cut, out.Note = true, fmt.Sprintf("read %d values and stopped; give a module (and a state) to narrow the answer", MaxBuildReads)
break
}
reads++
row, ok, err := r.builderRow(k)
if err != nil {
return nil, err
return BuildsAnswer{}, err
}
if !ok || (module != "" && row.Module != module) {
if !ok {
continue
}
if state != "" {
@@ -143,9 +261,26 @@ func (r Reader) BuilderBuilds(module, state string) ([]BuilderRow, error) {
continue
}
}
rows = append(rows, row)
out.Builds = append(out.Builds, row)
// Without a module the walk is in key order and stops at the limit; with one, all its (few) keys are read to order them.
if prefix == "" && len(out.Builds) >= limit {
if reads < len(candidates) {
out.Cut, out.Note = true, fmt.Sprintf("stopped at the limit of %d builds in key order, not newest first; give a module for its newest", limit)
}
break
}
}
return rows, nil
sort.SliceStable(out.Builds, func(i, j int) bool {
if out.Builds[i].Revision != out.Builds[j].Revision {
return out.Builds[i].Revision > out.Builds[j].Revision
}
return out.Builds[i].Key < out.Builds[j].Key
})
if len(out.Builds) > limit {
out.Builds = out.Builds[:limit]
out.Cut, out.Note = true, fmt.Sprintf("cut to the newest %d builds of %s", limit, module)
}
return out, nil
}
// BuilderBuild is the build of module at the new-form fingerprint fp2; with an empty fp2 it is the build at the fingerprint
@@ -156,6 +291,9 @@ func (r Reader) BuilderBuild(module, fp2, commit string) (BuilderRow, error) {
if r.Builds.Get == nil {
return BuilderRow{}, fmt.Errorf("the builder's builds cannot be read here")
}
if fp2 != "" && commit != "" {
return BuilderRow{}, fmt.Errorf("give fp2 or commit, not both: fp2 %s names a build directly, commit %s asks for the build at the indexer's fingerprint of that commit", fp2, commit)
}
if fp2 == "" {
raw, err := r.Module(module)
if err != nil {
+1 -1
View File
@@ -95,7 +95,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
}
// The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer
// itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads.
if want := []string{"indexer_module", "indexer_modules", "builder_build", "builder_builds"}; !reflect.DeepEqual(m.Tools, want) {
if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds"}; !reflect.DeepEqual(m.Tools, want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
for _, a := range m.Build.Artifacts {
+2
View File
@@ -6,6 +6,8 @@
"tools": [
"indexer_module",
"indexer_modules",
"indexer_candidate",
"indexer_candidates",
"builder_build",
"builder_builds"
],