Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d83156fb21 | ||
|
|
16ca51dc1f |
@@ -59,11 +59,52 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
kept, err := r.Kept(level, digest)
|
||||
return map[string]any{"kept": kept}, err
|
||||
}},
|
||||
{Name: check.PreviewsReadTool,
|
||||
Description: "The merge previews the checker holds, each as stored under previews.<repository>.<head>.<trunk>: the merge commit, per module directory " +
|
||||
"its tree, manifest digest and lock hashes, or the files in conflict. With a repository, that repository's; with a head (7 or more characters) " +
|
||||
"only that head's. Keys that are no preview key are listed as unreadable. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||
"head": map[string]any{"type": "string", "description": "a pull request head commit, or a prefix of at least 7 characters"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
repository, _ := args["repository"].(string)
|
||||
head, _ := args["head"].(string)
|
||||
if head != "" && len(head) < 7 {
|
||||
return nil, fmt.Errorf("the head %q is too short: give at least 7 characters", head)
|
||||
}
|
||||
ps, bad, err := r.Previews(repository, head)
|
||||
return map[string]any{"previews": ps, "unreadable": bad}, err
|
||||
}},
|
||||
{Name: check.StatusReadTool,
|
||||
Description: "What holds the checker back and what runs: the capacity per node (the setting, the default one step per node), whether the controller's " +
|
||||
"delivery plans in their build phase hold a key (no new check starts while one does), the steps in flight on the build modules by node, and the " +
|
||||
"candidates without a verdict for their heads. The steps the checker holds waiting for a place are in its process, not in a bucket, and the " +
|
||||
"answer says so. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return r.Status() }},
|
||||
}
|
||||
}
|
||||
|
||||
func capacity() (string, error) {
|
||||
path := os.Getenv("MESH_CHECKER_SETTINGS")
|
||||
if path == "" {
|
||||
return os.Getenv("MESH_CHECKER_CAPACITY"), nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var s struct {
|
||||
Capacity string `json:"capacity"`
|
||||
}
|
||||
return s.Capacity, json.Unmarshal(raw, &s)
|
||||
}
|
||||
|
||||
func main() {
|
||||
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates"))}
|
||||
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
||||
PreviewState: read(stdio.State("previews")), PlansState: read(stdio.State("mesh-controller.plans-building")),
|
||||
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
|
||||
Capacity: capacity}
|
||||
if err := stdio.Serve("", Tools(r)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
func TestTheToolsServedAreTheThreeReadTools(t *testing.T) {
|
||||
func TestTheToolsServedAreTheFiveReadTools(t *testing.T) {
|
||||
var names []string
|
||||
for _, tl := range Tools(check.Reader{}) {
|
||||
names = append(names, tl.Name)
|
||||
@@ -14,12 +15,15 @@ func TestTheToolsServedAreTheThreeReadTools(t *testing.T) {
|
||||
t.Errorf("%s has no description or no run", tl.Name)
|
||||
}
|
||||
}
|
||||
want := []string{check.VerdictsReadTool, check.PullReadTool, check.KeptReadTool}
|
||||
if len(names) != 3 || names[0] != want[0] || names[1] != want[1] || names[2] != want[2] {
|
||||
want := []string{check.VerdictsReadTool, check.PullReadTool, check.KeptReadTool, check.PreviewsReadTool, check.StatusReadTool}
|
||||
if !slices.Equal(names, want) {
|
||||
t.Errorf("tools = %v, want %v", names, want)
|
||||
}
|
||||
// With nothing to read from a tool says so and changes nothing.
|
||||
for _, tl := range Tools(check.Reader{}) {
|
||||
if tl.Name == check.StatusReadTool {
|
||||
continue // it answers from what it can read, and says what it could not
|
||||
}
|
||||
if _, err := tl.Run(map[string]any{"repository": "novox/x", "number": float64(1)}); err == nil {
|
||||
t.Errorf("%s answered with no state to read", tl.Name)
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the
|
||||
@@ -17,6 +18,9 @@ const (
|
||||
VerdictsReadTool = "checker_verdicts"
|
||||
PullReadTool = "checker_pull"
|
||||
KeptReadTool = "checker_kept"
|
||||
// PreviewsReadTool reads the previews state (issue 1680); StatusReadTool reads what the checker is held by and what it runs (issue 1685).
|
||||
PreviewsReadTool = "checker_previews"
|
||||
StatusReadTool = "checker_status"
|
||||
)
|
||||
|
||||
// StateRead reads one state: its keys and a key's stored value (nil for a key with none).
|
||||
@@ -26,7 +30,14 @@ type StateRead struct {
|
||||
}
|
||||
|
||||
// Reader answers the read tools from the checker's verdicts and kept states and the indexer's candidates.
|
||||
type Reader struct{ VerdictState, KeptState, CandidateState StateRead }
|
||||
type Reader struct {
|
||||
VerdictState, KeptState, CandidateState StateRead
|
||||
|
||||
PreviewState StateRead // the checker's merge previews
|
||||
PlansState StateRead // the controller's delivery plans in their build phase; zero where the bucket is absent
|
||||
StepStates map[string]StateRead // the build modules' steps states, by module
|
||||
Capacity func() (string, error) // the capacity setting as written, `node=steps,...`
|
||||
}
|
||||
|
||||
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
|
||||
// candidate still holds those heads; without it the report is that of the verdict alone.
|
||||
@@ -191,3 +202,153 @@ func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, error) {
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// PreviewRead is one preview as stored, with the parts of its key.
|
||||
type PreviewRead struct {
|
||||
Key string `json:"key"`
|
||||
Repository string `json:"repository"`
|
||||
Head string `json:"head"`
|
||||
Trunk string `json:"trunk"`
|
||||
Preview json.RawMessage `json:"preview"`
|
||||
}
|
||||
|
||||
// Previews are the previews the checker holds, each as stored. With a repository only that repository's; with a head (a
|
||||
// prefix of at least 7 characters is enough) only previews of that head. A key that is no preview key is listed as unreadable.
|
||||
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, error) {
|
||||
if r.PreviewState.Keys == nil || r.PreviewState.Get == nil {
|
||||
return nil, nil, fmt.Errorf("the previews state cannot be read here")
|
||||
}
|
||||
keys, err := r.PreviewState.Keys()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := []PreviewRead{}
|
||||
var bad []string
|
||||
for _, k := range keys {
|
||||
repo, h, trunk, err := delivery.ParsePreviewKey(k)
|
||||
if err != nil {
|
||||
bad = append(bad, k)
|
||||
continue
|
||||
}
|
||||
if (repository != "" && repo != repository) || (head != "" && !strings.HasPrefix(h, head)) {
|
||||
continue
|
||||
}
|
||||
raw, err := r.PreviewState.Get(k)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if len(raw) > 0 {
|
||||
out = append(out, PreviewRead{Key: k, Repository: repo, Head: h, Trunk: trunk, Preview: raw})
|
||||
}
|
||||
}
|
||||
return out, bad, nil
|
||||
}
|
||||
|
||||
// RunningStep is a step in flight on a build module.
|
||||
type RunningStep struct {
|
||||
Module string `json:"module"`
|
||||
Node string `json:"node"`
|
||||
ID string `json:"id"`
|
||||
Tool string `json:"tool"`
|
||||
Since string `json:"since,omitempty"`
|
||||
}
|
||||
|
||||
// Status is what holds the checker back and what runs now, read from the buckets: the checker's held and running steps live in
|
||||
// its process, so they are shown as what the buckets say (the steps in flight on the build modules, the candidates without a
|
||||
// verdict for their heads, and the delivery plans in their build phase), not as the process's own counters.
|
||||
type Status struct {
|
||||
Capacity map[string]int `json:"capacity"`
|
||||
CapacityNote string `json:"capacity_note,omitempty"`
|
||||
DefaultPerNode int `json:"default_per_node"`
|
||||
PlansBuilding struct {
|
||||
Readable bool `json:"readable"`
|
||||
Keys []string `json:"keys"`
|
||||
Holds bool `json:"holds"`
|
||||
} `json:"plans_building"`
|
||||
Running []RunningStep `json:"running_steps"`
|
||||
RunningByNode map[string]int `json:"running_by_node"`
|
||||
Unjudged []string `json:"unjudged_candidates"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// Status reads the capacity, the plan bucket, the build modules' steps and the candidates against their verdicts.
|
||||
func (r Reader) Status() (Status, error) {
|
||||
st := Status{DefaultPerNode: buildrun.DefaultCapacity, Capacity: map[string]int{}, RunningByNode: map[string]int{}}
|
||||
if r.Capacity == nil {
|
||||
st.CapacityNote = "the capacity setting cannot be read from here"
|
||||
} else if setting, err := r.Capacity(); err != nil {
|
||||
st.CapacityNote = "the capacity setting cannot be read: " + err.Error()
|
||||
} else if st.Capacity, err = buildrun.ParseCapacity(setting); err != nil {
|
||||
st.Capacity, st.CapacityNote = map[string]int{}, "the capacity setting is not valid: "+err.Error()
|
||||
}
|
||||
if r.PlansState.Keys == nil {
|
||||
st.Notes = append(st.Notes, PlansBuildingBucket+" is not readable here (absent), so it is read as empty: no delivery holds the checker")
|
||||
} else if keys, err := r.PlansState.Keys(); err != nil {
|
||||
st.Notes = append(st.Notes, PlansBuildingBucket+" cannot be listed, so it is read as empty: "+err.Error())
|
||||
} else {
|
||||
sort.Strings(keys)
|
||||
st.PlansBuilding.Readable, st.PlansBuilding.Keys, st.PlansBuilding.Holds = true, keys, len(keys) > 0
|
||||
}
|
||||
mods := make([]string, 0, len(r.StepStates))
|
||||
for m := range r.StepStates {
|
||||
mods = append(mods, m)
|
||||
}
|
||||
sort.Strings(mods)
|
||||
st.Running = []RunningStep{}
|
||||
for _, m := range mods {
|
||||
sr := r.StepStates[m]
|
||||
if sr.Keys == nil || sr.Get == nil {
|
||||
continue
|
||||
}
|
||||
keys, err := sr.Keys()
|
||||
if err != nil {
|
||||
st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error())
|
||||
continue
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
raw, err := sr.Get(k)
|
||||
if err != nil || len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
step, err := delivery.DecodeStep(raw)
|
||||
if err != nil || step.State != delivery.StepRunning {
|
||||
continue
|
||||
}
|
||||
rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool}
|
||||
if !step.Started.IsZero() {
|
||||
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
|
||||
}
|
||||
st.Running = append(st.Running, rs)
|
||||
st.RunningByNode[step.Node]++
|
||||
}
|
||||
}
|
||||
st.Unjudged = []string{}
|
||||
if r.CandidateState.Keys != nil && r.CandidateState.Get != nil && r.VerdictState.Get != nil {
|
||||
keys, err := r.CandidateState.Keys()
|
||||
if err != nil {
|
||||
return st, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
raw, err := r.CandidateState.Get(k)
|
||||
if err != nil || len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
c, err := delivery.DecodeCandidate(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if v, err := r.VerdictState.Get(vkey); err == nil && len(v) == 0 {
|
||||
st.Unjudged = append(st.Unjudged, c.Group)
|
||||
}
|
||||
}
|
||||
}
|
||||
st.Notes = append(st.Notes, "the steps the checker holds waiting for a place are in its process and in no bucket: a candidate without a verdict while a plan key exists, or while its node runs as many steps as its capacity, is held")
|
||||
return st, nil
|
||||
}
|
||||
|
||||
@@ -3,8 +3,10 @@ package check
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -102,3 +104,64 @@ func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { //
|
||||
t.Error("a digest nothing kept was answered")
|
||||
}
|
||||
}
|
||||
|
||||
// 1680 live acceptance: checker_previews shows the preview key of an open pull request, as stored, narrowed by repository and head.
|
||||
func TestThePreviewsToolShowsTheStoredPreviewOfAPullRequest(t *testing.T) {
|
||||
pv := delivery.Preview{Repository: repoA, Head: head1, Trunk: head2, Merge: "m1", Modules: map[string]delivery.PreviewModule{}}
|
||||
data, err := delivery.EncodePreview(pv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.PreviewKey(repoA, head1, head2)
|
||||
rd := Reader{PreviewState: fakeState{key: data, "junk": []byte("{}")}.read()}
|
||||
got, bad, err := rd.Previews(repoA, head1[:8])
|
||||
if err != nil || len(got) != 1 || got[0].Key != key || got[0].Head != head1 || got[0].Trunk != head2 || string(got[0].Preview) != string(data) {
|
||||
t.Fatalf("previews = %+v, %v", got, err)
|
||||
}
|
||||
if len(bad) != 1 || bad[0] != "junk" {
|
||||
t.Errorf("unreadable = %v", bad)
|
||||
}
|
||||
if got, _, _ := rd.Previews("novox/other", ""); len(got) != 0 {
|
||||
t.Errorf("another repository's = %+v", got)
|
||||
}
|
||||
if _, _, err := (Reader{}).Previews("", ""); err == nil {
|
||||
t.Error("answered with no state to read")
|
||||
}
|
||||
}
|
||||
|
||||
// 1685 live acceptance: checker_status reads the capacity, whether a plan key holds the checker, the steps in flight by node and
|
||||
// the candidates without a verdict.
|
||||
func TestTheStatusToolReadsCapacityHoldsAndRunningSteps(t *testing.T) {
|
||||
running, _ := delivery.EncodeStep(delivery.Step{ID: "abc1", Tool: "build_go_test", Node: "shanks", State: delivery.StepRunning, Started: time.Date(2026, 10, 22, 15, 0, 0, 0, time.UTC)})
|
||||
passed, _ := delivery.EncodeStep(delivery.Step{ID: "abc2", Tool: "build_go_test", Node: "ace", State: delivery.StepPassed})
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
ckey, _ := delivery.CandidatesKey("feat/x")
|
||||
rd := Reader{
|
||||
Capacity: func() (string, error) { return "shanks=2,ace=0", nil },
|
||||
PlansState: fakeState{"p1": []byte("{}")}.read(),
|
||||
StepStates: map[string]StateRead{"mesh-build-go": fakeState{"shanks.abc1": running, "ace.abc2": passed}.read()},
|
||||
CandidateState: fakeState{ckey: cand}.read(),
|
||||
VerdictState: fakeState{}.read(),
|
||||
}
|
||||
st, err := rd.Status()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st.Capacity["shanks"] != 2 || st.Capacity["ace"] != 0 || st.DefaultPerNode != 1 {
|
||||
t.Errorf("capacity = %v default %d", st.Capacity, st.DefaultPerNode)
|
||||
}
|
||||
if !st.PlansBuilding.Holds || len(st.PlansBuilding.Keys) != 1 {
|
||||
t.Errorf("plans = %+v", st.PlansBuilding)
|
||||
}
|
||||
if len(st.Running) != 1 || st.Running[0].Node != "shanks" || st.RunningByNode["shanks"] != 1 || st.RunningByNode["ace"] != 0 {
|
||||
t.Errorf("running = %+v %v", st.Running, st.RunningByNode)
|
||||
}
|
||||
if len(st.Unjudged) != 1 || st.Unjudged[0] != "feat/x" {
|
||||
t.Errorf("unjudged = %v", st.Unjudged)
|
||||
}
|
||||
// An absent plan bucket is read as empty and said; an unreadable capacity is said, not guessed.
|
||||
st, err = Reader{Capacity: func() (string, error) { return "", os.ErrNotExist }}.Status()
|
||||
if err != nil || st.PlansBuilding.Holds || st.CapacityNote == "" || len(st.Notes) == 0 {
|
||||
t.Errorf("empty status = %+v, %v", st, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -118,7 +118,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
||||
t.Errorf("streams = %+v, want %+v", m.Streams, want)
|
||||
}
|
||||
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
|
||||
if want := []string{"checker_kept", "checker_pull", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats", "bus"} {
|
||||
|
||||
@@ -31,7 +31,9 @@
|
||||
],
|
||||
"tools": [
|
||||
"checker_kept",
|
||||
"checker_previews",
|
||||
"checker_pull",
|
||||
"checker_status",
|
||||
"checker_verdicts"
|
||||
],
|
||||
"invokes": [
|
||||
|
||||
@@ -35,18 +35,39 @@ func Tools(r view.Reader) []stdio.Tool {
|
||||
mods, err := r.Modules(only)
|
||||
return map[string]any{"modules": mods}, err
|
||||
}},
|
||||
{Name: view.CandidateReadTool,
|
||||
Description: "What the indexer stored for one candidate (the open pull requests of a branch) under candidates.<group>, as stored: " +
|
||||
"the pull requests and heads, the base, and per pull request the touched modules with their new-form fingerprint (fp2) or why " +
|
||||
"they have none (none2). Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"group"}, "properties": map[string]any{
|
||||
"group": map[string]any{"type": "string", "description": "the candidate's group, the branch name its pull requests share"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
group, _ := args["group"].(string)
|
||||
return r.Candidate(group)
|
||||
}},
|
||||
{Name: view.CandidatesReadTool,
|
||||
Description: "Every candidate the indexer holds, by group, each as stored under candidates.<group>; with module, only those with a " +
|
||||
"pull request touching that module. Keys that cannot be read are listed as unreadable. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "only candidates touching this module"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
module, _ := args["module"].(string)
|
||||
c, bad, err := r.Candidates(module)
|
||||
return map[string]any{"candidates": c, "unreadable": bad}, err
|
||||
}},
|
||||
{Name: view.BuilderBuildsReadTool,
|
||||
Description: "The builder's build keys, each as stored in mesh-delivery-builder_builds: key, module, fingerprint (src2) and build " +
|
||||
"(state, by, step with the nodes it was built on, manifest, artifacts with their digests, toolchains, error). Narrow by module " +
|
||||
"or state (queued, building, done, failed). Read-only.",
|
||||
"or state (queued, building, done, failed); at most limit builds (default 50), newest revision first for a module, and the answer says when it was cut and lists keys that are no build key (unreadable). Without a module, give one to get its newest. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "only this module's builds"},
|
||||
"state": map[string]any{"type": "string", "description": "only builds in this state"}}},
|
||||
"state": map[string]any{"type": "string", "description": "only builds in this state"},
|
||||
"limit": map[string]any{"type": "integer", "description": "at most this many builds (default 50), newest revision first"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
module, _ := args["module"].(string)
|
||||
state, _ := args["state"].(string)
|
||||
rows, err := r.BuilderBuilds(module, state)
|
||||
return map[string]any{"builds": rows}, err
|
||||
limit, _ := args["limit"].(float64)
|
||||
return r.BuilderBuilds(module, state, int(limit))
|
||||
}},
|
||||
{Name: view.BuilderBuildReadTool,
|
||||
Description: "One build of the builder as stored. Give the module and its fingerprint (fp2, src2:...), or the module alone for the build " +
|
||||
@@ -65,7 +86,13 @@ func Tools(r view.Reader) []stdio.Tool {
|
||||
}
|
||||
|
||||
func read(state stdio.KeptState) view.StateRead {
|
||||
return view.StateRead{Keys: state.Keys, Get: func(key string) (json.RawMessage, error) {
|
||||
return view.StateRead{Keys: state.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) {
|
||||
e, err := state.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return e.Value, e.Revision, nil
|
||||
}, Get: func(key string) (json.RawMessage, error) {
|
||||
e, err := state.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, err
|
||||
@@ -75,7 +102,7 @@ func read(state stdio.KeptState) view.StateRead {
|
||||
}
|
||||
|
||||
func main() {
|
||||
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds"))}
|
||||
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates"))}
|
||||
if err := stdio.Serve("", Tools(r)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -117,13 +118,16 @@ func TestBuilderBuildsListsKeysAsStored(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rows := out.(map[string]any)["builds"].([]view.BuilderRow)
|
||||
rows := out.(view.BuildsAnswer).Builds
|
||||
if u := out.(view.BuildsAnswer).Unreadable; len(u) != 1 || u[0] != "junk" {
|
||||
t.Errorf("unreadable = %v, want [junk] (issue 1818)", u)
|
||||
}
|
||||
if len(rows) != 2 || rows[0].Key != keyA || rows[0].Module != "netcheck" || rows[0].Fingerprint != fp2A || string(rows[0].Build) != doneBuild || string(rows[1].Build) != failBuild {
|
||||
t.Errorf("rows = %+v", rows)
|
||||
}
|
||||
for args, want := range map[string]string{"state": keyB, "module": keyA} {
|
||||
out, _ := run(t, r, view.BuilderBuildsReadTool, map[string]any{args: map[string]string{keyB: "failed", keyA: "netcheck"}[want]})
|
||||
got := out.(map[string]any)["builds"].([]view.BuilderRow)
|
||||
got := out.(view.BuildsAnswer).Builds
|
||||
if len(got) != 1 || got[0].Key != want {
|
||||
t.Errorf("%s filter = %+v, want %s", args, got, want)
|
||||
}
|
||||
@@ -156,3 +160,107 @@ func TestBuilderBuildJoinsTheIndexersFingerprint(t *testing.T) {
|
||||
t.Errorf("an unbuilt fingerprint: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// bigBuilds is a bucket of n keys over 100 modules that counts the values read.
|
||||
type bigBuilds struct {
|
||||
fake fake
|
||||
reads *int
|
||||
}
|
||||
|
||||
func newBig(n int) bigBuilds {
|
||||
f := fake{}
|
||||
for i := 0; i < n; i++ {
|
||||
f[fmt.Sprintf("mod%02d.%064x", i%100, i)] = doneBuild
|
||||
}
|
||||
return bigBuilds{fake: f, reads: new(int)}
|
||||
}
|
||||
|
||||
func (b bigBuilds) reader() view.Reader {
|
||||
r := b.fake.reader()
|
||||
get := r.State.Get
|
||||
r.Builds = view.StateRead{Keys: r.State.Keys, Get: func(k string) (json.RawMessage, error) { *b.reads++; return get(k) },
|
||||
GetEntry: func(k string) (json.RawMessage, uint64, error) {
|
||||
*b.reads++
|
||||
raw, err := get(k)
|
||||
var rev uint64
|
||||
fmt.Sscanf(k[strings.Index(k, ".")+1:], "%x", &rev) // the fake's revision is the key's number
|
||||
return raw, rev + 1, err
|
||||
}}
|
||||
return r
|
||||
}
|
||||
|
||||
// 1827/AC1: with 2,000 keys, a module's builds are found by key prefix, so only that module's values are read (20, not 2,000),
|
||||
// newest revision first, within the limit, and the answer says it was cut.
|
||||
func TestBuilderBuildsFiltersByPrefixBeforeAnyGet(t *testing.T) {
|
||||
b := newBig(2000)
|
||||
out, err := run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{"module": "mod07", "limit": float64(5)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := out.(view.BuildsAnswer)
|
||||
if *b.reads != 20 {
|
||||
t.Errorf("read %d values for a module of 20 keys, want 20", *b.reads)
|
||||
}
|
||||
if len(a.Builds) != 5 || !a.Cut || !strings.Contains(a.Note, "5") {
|
||||
t.Errorf("answer = %d rows, cut %v, note %q", len(a.Builds), a.Cut, a.Note)
|
||||
}
|
||||
for i := 1; i < len(a.Builds); i++ {
|
||||
if a.Builds[i-1].Revision < a.Builds[i].Revision {
|
||||
t.Errorf("not newest first: %d before %d", a.Builds[i-1].Revision, a.Builds[i].Revision)
|
||||
}
|
||||
}
|
||||
if a.Builds[0].Revision != 1908 {
|
||||
t.Errorf("newest revision = %d, want 1908", a.Builds[0].Revision)
|
||||
}
|
||||
}
|
||||
|
||||
// 1827/AC2: with no module the answer reads at most the limit's worth in key order (default 50), never the 2,000.
|
||||
func TestBuilderBuildsWithoutAModuleIsBounded(t *testing.T) {
|
||||
b := newBig(2000)
|
||||
out, err := run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := out.(view.BuildsAnswer)
|
||||
if len(a.Builds) != view.DefaultBuildsLimit || !a.Cut || *b.reads > view.DefaultBuildsLimit {
|
||||
t.Errorf("%d rows, cut %v, %d reads", len(a.Builds), a.Cut, *b.reads)
|
||||
}
|
||||
b = newBig(2000)
|
||||
out, _ = run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{"state": "failed"})
|
||||
if a := out.(view.BuildsAnswer); *b.reads != view.MaxBuildReads || !a.Cut || len(a.Builds) != 0 {
|
||||
t.Errorf("a state matching nothing: %d reads (want the bound %d), cut %v", *b.reads, view.MaxBuildReads, a.Cut)
|
||||
}
|
||||
}
|
||||
|
||||
// 1818: fp2 together with commit is refused, naming both.
|
||||
func TestBuilderBuildRefusesFP2WithCommit(t *testing.T) {
|
||||
r := both(fake{keyA: doneBuild}, fake{"netcheck": `{"v":3,"fp2":"` + fp2A + `","commit":"c1","path":"."}`})
|
||||
_, err := run(t, r, view.BuilderBuildReadTool, map[string]any{"module": "netcheck", "fp2": fp2A, "commit": "c1"})
|
||||
if err == nil || !strings.Contains(err.Error(), "fp2") || !strings.Contains(err.Error(), "commit") || !strings.Contains(err.Error(), "not both") {
|
||||
t.Errorf("err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1681: indexer_candidates and indexer_candidate answer the stored value; a pull request's fp2 per touched module shows.
|
||||
func TestCandidateToolsAnswerTheStoredValue(t *testing.T) {
|
||||
cand := `{"v":1,"group":"feat/x","base":3,"heads":"h","pulls":[{"modules":["zsh"],"fps":{"zsh":{"fp2":"` + fp2B + `","manifest":"m"}}}]}`
|
||||
other := `{"v":1,"group":"fix","base":3,"heads":"h","pulls":[{"modules":["netcheck"]}]}`
|
||||
r := fake{"feat/x": cand, "fix": other, "bad": "{"}.reader()
|
||||
r.CandidatesState = r.State
|
||||
out, err := run(t, r, view.CandidateReadTool, map[string]any{"group": "feat/x"})
|
||||
if err != nil || string(out.(json.RawMessage)) != cand || !strings.Contains(string(out.(json.RawMessage)), fp2B) {
|
||||
t.Fatalf("candidate = %v, %v", out, err)
|
||||
}
|
||||
if _, err := run(t, r, view.CandidateReadTool, map[string]any{"group": "gone"}); err == nil || !strings.Contains(err.Error(), "no candidate gone") {
|
||||
t.Errorf("err = %v", err)
|
||||
}
|
||||
out, err = run(t, r, view.CandidatesReadTool, map[string]any{})
|
||||
m := out.(map[string]any)
|
||||
if err != nil || len(m["candidates"].(map[string]json.RawMessage)) != 2 || len(m["unreadable"].([]string)) != 1 {
|
||||
t.Errorf("all = %v, %v", m, err)
|
||||
}
|
||||
out, _ = run(t, r, view.CandidatesReadTool, map[string]any{"module": "zsh"})
|
||||
if c := out.(map[string]any)["candidates"].(map[string]json.RawMessage); len(c) != 1 || c["feat/x"] == nil {
|
||||
t.Errorf("by module = %v", c)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ package view
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -23,12 +24,16 @@ const (
|
||||
type StateRead struct {
|
||||
Keys func() ([]string, error)
|
||||
Get func(key string) (json.RawMessage, error)
|
||||
// GetEntry, when set, also gives the revision of the value; builder_builds orders by it.
|
||||
GetEntry func(key string) (json.RawMessage, uint64, error)
|
||||
}
|
||||
|
||||
// Reader answers the read tools from a modules state.
|
||||
type Reader struct {
|
||||
State StateRead // the indexer's modules
|
||||
Builds StateRead // the builder's builds
|
||||
|
||||
CandidatesState StateRead // the indexer's candidates
|
||||
}
|
||||
|
||||
// Module is the stored value of modules.<module>, exactly as the indexer wrote it.
|
||||
@@ -84,6 +89,67 @@ func (r Reader) Modules(none2 bool) (map[string]json.RawMessage, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// The read tools of the indexer's candidates (issue 1681): what the indexer stored under candidates.<group>, as stored, so the
|
||||
// new-form fingerprint of each touched module of an open pull request (pulls[].fps) can be read live.
|
||||
const (
|
||||
CandidateReadTool = "indexer_candidate"
|
||||
CandidatesReadTool = "indexer_candidates"
|
||||
)
|
||||
|
||||
// Candidate is the stored value of the candidates key of a group, exactly as the indexer wrote it.
|
||||
func (r Reader) Candidate(group string) (json.RawMessage, error) {
|
||||
key, err := delivery.CandidatesKey(strings.TrimSpace(group))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.CandidatesState.Get == nil {
|
||||
return nil, fmt.Errorf("the candidates state cannot be read here")
|
||||
}
|
||||
raw, err := r.CandidatesState.Get(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, fmt.Errorf("no candidate %s in the candidates state", group)
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
// Candidates is every candidate, by group, each as stored; with a module only the candidates with a pull request touching it.
|
||||
// A value that is not readable is listed under unreadable with its key, not skipped.
|
||||
func (r Reader) Candidates(module string) (map[string]json.RawMessage, []string, error) {
|
||||
if r.CandidatesState.Keys == nil || r.CandidatesState.Get == nil {
|
||||
return nil, nil, fmt.Errorf("the candidates state cannot be read here")
|
||||
}
|
||||
keys, err := r.CandidatesState.Keys()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := map[string]json.RawMessage{}
|
||||
var unreadable []string
|
||||
for _, k := range keys {
|
||||
raw, err := r.CandidatesState.Get(k)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
group, perr := delivery.ParseCandidatesKey(k)
|
||||
var c delivery.Candidate
|
||||
if perr != nil || json.Unmarshal(raw, &c) != nil {
|
||||
unreadable = append(unreadable, k)
|
||||
continue
|
||||
}
|
||||
if module != "" && !slices.ContainsFunc(c.Pulls, func(p delivery.CandidatePull) bool { return slices.Contains(p.Modules, module) }) {
|
||||
continue
|
||||
}
|
||||
out[group] = raw
|
||||
}
|
||||
return out, unreadable, nil
|
||||
}
|
||||
|
||||
// The read tools of the builder's builds (novox/hq issue 1723): what the builder stored under `<module>.<fingerprint hex>`
|
||||
// in mesh-delivery-builder_builds, answered as stored, so its states, nodes (in the step) and digests can be checked live.
|
||||
const (
|
||||
@@ -91,48 +157,100 @@ const (
|
||||
BuilderBuildsReadTool = "builder_builds"
|
||||
)
|
||||
|
||||
// BuilderRow is one key of the builder's builds: the key, the module and the fingerprint it parses to, and the stored value
|
||||
// as the builder wrote it (state, by, step, manifest, artifacts with digests, toolchains, error).
|
||||
// BuilderRow is one key of the builder's builds: the key, the module and the fingerprint it parses to, the revision of its
|
||||
// stored value (when the read gives one) and the stored value as the builder wrote it.
|
||||
type BuilderRow struct {
|
||||
Key string `json:"key"`
|
||||
Module string `json:"module"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Revision uint64 `json:"revision,omitempty"`
|
||||
Build json.RawMessage `json:"build"`
|
||||
}
|
||||
|
||||
// The bounds of builder_builds (issue 1827): the rows it returns when no limit is given, and the values it reads in one call.
|
||||
// Reading a value is a bus round trip, so the answer must not read the whole bucket (it timed out at 30 s on the live one).
|
||||
const (
|
||||
DefaultBuildsLimit = 50
|
||||
MaxBuildReads = 200
|
||||
)
|
||||
|
||||
// BuildsAnswer is the answer of builder_builds: the rows, newest revision first, whether the answer was cut and why, and
|
||||
// the keys that are no build key (Unreadable), which are listed rather than silently skipped (issue 1818).
|
||||
type BuildsAnswer struct {
|
||||
Builds []BuilderRow `json:"builds"`
|
||||
Cut bool `json:"cut,omitempty"`
|
||||
Note string `json:"note,omitempty"`
|
||||
Unreadable []string `json:"unreadable,omitempty"`
|
||||
}
|
||||
|
||||
func (r Reader) get(key string) (json.RawMessage, uint64, error) {
|
||||
if r.Builds.GetEntry != nil {
|
||||
return r.Builds.GetEntry(key)
|
||||
}
|
||||
raw, err := r.Builds.Get(key)
|
||||
return raw, 0, err
|
||||
}
|
||||
|
||||
func (r Reader) builderRow(key string) (BuilderRow, bool, error) {
|
||||
module, fp, err := delivery.ParseBuilderBuildKey(key)
|
||||
if err != nil {
|
||||
return BuilderRow{}, false, nil // a key that is no build key is not a build
|
||||
}
|
||||
raw, err := r.Builds.Get(key)
|
||||
raw, rev, err := r.get(key)
|
||||
if err != nil {
|
||||
return BuilderRow{}, false, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return BuilderRow{}, false, nil
|
||||
}
|
||||
return BuilderRow{Key: key, Module: module, Fingerprint: fp, Build: raw}, true, nil
|
||||
return BuilderRow{Key: key, Module: module, Fingerprint: fp, Revision: rev, Build: raw}, true, nil
|
||||
}
|
||||
|
||||
// BuilderBuilds lists the builder's keys, each as stored, sorted by key; module and state, when given, keep only the builds
|
||||
// of that module or in that state.
|
||||
func (r Reader) BuilderBuilds(module, state string) ([]BuilderRow, error) {
|
||||
// BuilderBuilds lists the builder's keys, each as stored, newest revision first. With a module it considers only the keys
|
||||
// that begin `<module>.`, before any value is read; with a state it keeps the builds in that state. It returns at most limit
|
||||
// rows (DefaultBuildsLimit when limit is not above zero) and reads at most MaxBuildReads values, and says when it cut the
|
||||
// answer. Keys that are no build key are listed as unreadable. Without a module the keys are walked in key order, so the
|
||||
// answer is the first rows found, not the newest of the bucket: give a module to get its newest.
|
||||
func (r Reader) BuilderBuilds(module, state string, limit int) (BuildsAnswer, error) {
|
||||
if r.Builds.Keys == nil || r.Builds.Get == nil {
|
||||
return nil, fmt.Errorf("the builder's builds cannot be read here")
|
||||
return BuildsAnswer{}, fmt.Errorf("the builder's builds cannot be read here")
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = DefaultBuildsLimit
|
||||
}
|
||||
keys, err := r.Builds.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return BuildsAnswer{}, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
rows := []BuilderRow{}
|
||||
prefix := ""
|
||||
if module = strings.TrimSpace(module); module != "" {
|
||||
prefix = module + "."
|
||||
}
|
||||
out := BuildsAnswer{Builds: []BuilderRow{}}
|
||||
var candidates []string
|
||||
for _, k := range keys {
|
||||
if prefix != "" && !strings.HasPrefix(k, prefix) {
|
||||
continue
|
||||
}
|
||||
if _, _, err := delivery.ParseBuilderBuildKey(k); err != nil {
|
||||
out.Unreadable = append(out.Unreadable, k)
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, k)
|
||||
}
|
||||
reads := 0
|
||||
for _, k := range candidates {
|
||||
if reads >= MaxBuildReads {
|
||||
out.Cut, out.Note = true, fmt.Sprintf("read %d values and stopped; give a module (and a state) to narrow the answer", MaxBuildReads)
|
||||
break
|
||||
}
|
||||
reads++
|
||||
row, ok, err := r.builderRow(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return BuildsAnswer{}, err
|
||||
}
|
||||
if !ok || (module != "" && row.Module != module) {
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if state != "" {
|
||||
@@ -143,9 +261,26 @@ func (r Reader) BuilderBuilds(module, state string) ([]BuilderRow, error) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
rows = append(rows, row)
|
||||
out.Builds = append(out.Builds, row)
|
||||
// Without a module the walk is in key order and stops at the limit; with one, all its (few) keys are read to order them.
|
||||
if prefix == "" && len(out.Builds) >= limit {
|
||||
if reads < len(candidates) {
|
||||
out.Cut, out.Note = true, fmt.Sprintf("stopped at the limit of %d builds in key order, not newest first; give a module for its newest", limit)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return rows, nil
|
||||
sort.SliceStable(out.Builds, func(i, j int) bool {
|
||||
if out.Builds[i].Revision != out.Builds[j].Revision {
|
||||
return out.Builds[i].Revision > out.Builds[j].Revision
|
||||
}
|
||||
return out.Builds[i].Key < out.Builds[j].Key
|
||||
})
|
||||
if len(out.Builds) > limit {
|
||||
out.Builds = out.Builds[:limit]
|
||||
out.Cut, out.Note = true, fmt.Sprintf("cut to the newest %d builds of %s", limit, module)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// BuilderBuild is the build of module at the new-form fingerprint fp2; with an empty fp2 it is the build at the fingerprint
|
||||
@@ -156,6 +291,9 @@ func (r Reader) BuilderBuild(module, fp2, commit string) (BuilderRow, error) {
|
||||
if r.Builds.Get == nil {
|
||||
return BuilderRow{}, fmt.Errorf("the builder's builds cannot be read here")
|
||||
}
|
||||
if fp2 != "" && commit != "" {
|
||||
return BuilderRow{}, fmt.Errorf("give fp2 or commit, not both: fp2 %s names a build directly, commit %s asks for the build at the indexer's fingerprint of that commit", fp2, commit)
|
||||
}
|
||||
if fp2 == "" {
|
||||
raw, err := r.Module(module)
|
||||
if err != nil {
|
||||
|
||||
@@ -95,7 +95,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
|
||||
}
|
||||
// The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer
|
||||
// itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads.
|
||||
if want := []string{"indexer_module", "indexer_modules", "builder_build", "builder_builds"}; !reflect.DeepEqual(m.Tools, want) {
|
||||
if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds"}; !reflect.DeepEqual(m.Tools, want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
|
||||
@@ -6,6 +6,8 @@
|
||||
"tools": [
|
||||
"indexer_module",
|
||||
"indexer_modules",
|
||||
"indexer_candidate",
|
||||
"indexer_candidates",
|
||||
"builder_build",
|
||||
"builder_builds"
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user