Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d83156fb21 | ||
|
|
16ca51dc1f |
@@ -59,11 +59,52 @@ func Tools(r check.Reader) []stdio.Tool {
|
|||||||
kept, err := r.Kept(level, digest)
|
kept, err := r.Kept(level, digest)
|
||||||
return map[string]any{"kept": kept}, err
|
return map[string]any{"kept": kept}, err
|
||||||
}},
|
}},
|
||||||
|
{Name: check.PreviewsReadTool,
|
||||||
|
Description: "The merge previews the checker holds, each as stored under previews.<repository>.<head>.<trunk>: the merge commit, per module directory " +
|
||||||
|
"its tree, manifest digest and lock hashes, or the files in conflict. With a repository, that repository's; with a head (7 or more characters) " +
|
||||||
|
"only that head's. Keys that are no preview key are listed as unreadable. Read-only.",
|
||||||
|
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||||
|
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||||
|
"head": map[string]any{"type": "string", "description": "a pull request head commit, or a prefix of at least 7 characters"}}},
|
||||||
|
Run: func(args map[string]any) (any, error) {
|
||||||
|
repository, _ := args["repository"].(string)
|
||||||
|
head, _ := args["head"].(string)
|
||||||
|
if head != "" && len(head) < 7 {
|
||||||
|
return nil, fmt.Errorf("the head %q is too short: give at least 7 characters", head)
|
||||||
|
}
|
||||||
|
ps, bad, err := r.Previews(repository, head)
|
||||||
|
return map[string]any{"previews": ps, "unreadable": bad}, err
|
||||||
|
}},
|
||||||
|
{Name: check.StatusReadTool,
|
||||||
|
Description: "What holds the checker back and what runs: the capacity per node (the setting, the default one step per node), whether the controller's " +
|
||||||
|
"delivery plans in their build phase hold a key (no new check starts while one does), the steps in flight on the build modules by node, and the " +
|
||||||
|
"candidates without a verdict for their heads. The steps the checker holds waiting for a place are in its process, not in a bucket, and the " +
|
||||||
|
"answer says so. Read-only.",
|
||||||
|
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||||
|
Run: func(map[string]any) (any, error) { return r.Status() }},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func capacity() (string, error) {
|
||||||
|
path := os.Getenv("MESH_CHECKER_SETTINGS")
|
||||||
|
if path == "" {
|
||||||
|
return os.Getenv("MESH_CHECKER_CAPACITY"), nil
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
var s struct {
|
||||||
|
Capacity string `json:"capacity"`
|
||||||
|
}
|
||||||
|
return s.Capacity, json.Unmarshal(raw, &s)
|
||||||
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates"))}
|
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
||||||
|
PreviewState: read(stdio.State("previews")), PlansState: read(stdio.State("mesh-controller.plans-building")),
|
||||||
|
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
|
||||||
|
Capacity: capacity}
|
||||||
if err := stdio.Serve("", Tools(r)); err != nil {
|
if err := stdio.Serve("", Tools(r)); err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
|
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"slices"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"mesh-delivery-checker/internal/check"
|
"mesh-delivery-checker/internal/check"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestTheToolsServedAreTheThreeReadTools(t *testing.T) {
|
func TestTheToolsServedAreTheFiveReadTools(t *testing.T) {
|
||||||
var names []string
|
var names []string
|
||||||
for _, tl := range Tools(check.Reader{}) {
|
for _, tl := range Tools(check.Reader{}) {
|
||||||
names = append(names, tl.Name)
|
names = append(names, tl.Name)
|
||||||
@@ -14,12 +15,15 @@ func TestTheToolsServedAreTheThreeReadTools(t *testing.T) {
|
|||||||
t.Errorf("%s has no description or no run", tl.Name)
|
t.Errorf("%s has no description or no run", tl.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
want := []string{check.VerdictsReadTool, check.PullReadTool, check.KeptReadTool}
|
want := []string{check.VerdictsReadTool, check.PullReadTool, check.KeptReadTool, check.PreviewsReadTool, check.StatusReadTool}
|
||||||
if len(names) != 3 || names[0] != want[0] || names[1] != want[1] || names[2] != want[2] {
|
if !slices.Equal(names, want) {
|
||||||
t.Errorf("tools = %v, want %v", names, want)
|
t.Errorf("tools = %v, want %v", names, want)
|
||||||
}
|
}
|
||||||
// With nothing to read from a tool says so and changes nothing.
|
// With nothing to read from a tool says so and changes nothing.
|
||||||
for _, tl := range Tools(check.Reader{}) {
|
for _, tl := range Tools(check.Reader{}) {
|
||||||
|
if tl.Name == check.StatusReadTool {
|
||||||
|
continue // it answers from what it can read, and says what it could not
|
||||||
|
}
|
||||||
if _, err := tl.Run(map[string]any{"repository": "novox/x", "number": float64(1)}); err == nil {
|
if _, err := tl.Run(map[string]any{"repository": "novox/x", "number": float64(1)}); err == nil {
|
||||||
t.Errorf("%s answered with no state to read", tl.Name)
|
t.Errorf("%s answered with no state to read", tl.Name)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the
|
// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the
|
||||||
@@ -17,6 +18,9 @@ const (
|
|||||||
VerdictsReadTool = "checker_verdicts"
|
VerdictsReadTool = "checker_verdicts"
|
||||||
PullReadTool = "checker_pull"
|
PullReadTool = "checker_pull"
|
||||||
KeptReadTool = "checker_kept"
|
KeptReadTool = "checker_kept"
|
||||||
|
// PreviewsReadTool reads the previews state (issue 1680); StatusReadTool reads what the checker is held by and what it runs (issue 1685).
|
||||||
|
PreviewsReadTool = "checker_previews"
|
||||||
|
StatusReadTool = "checker_status"
|
||||||
)
|
)
|
||||||
|
|
||||||
// StateRead reads one state: its keys and a key's stored value (nil for a key with none).
|
// StateRead reads one state: its keys and a key's stored value (nil for a key with none).
|
||||||
@@ -26,7 +30,14 @@ type StateRead struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Reader answers the read tools from the checker's verdicts and kept states and the indexer's candidates.
|
// Reader answers the read tools from the checker's verdicts and kept states and the indexer's candidates.
|
||||||
type Reader struct{ VerdictState, KeptState, CandidateState StateRead }
|
type Reader struct {
|
||||||
|
VerdictState, KeptState, CandidateState StateRead
|
||||||
|
|
||||||
|
PreviewState StateRead // the checker's merge previews
|
||||||
|
PlansState StateRead // the controller's delivery plans in their build phase; zero where the bucket is absent
|
||||||
|
StepStates map[string]StateRead // the build modules' steps states, by module
|
||||||
|
Capacity func() (string, error) // the capacity setting as written, `node=steps,...`
|
||||||
|
}
|
||||||
|
|
||||||
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
|
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
|
||||||
// candidate still holds those heads; without it the report is that of the verdict alone.
|
// candidate still holds those heads; without it the report is that of the verdict alone.
|
||||||
@@ -191,3 +202,153 @@ func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, error) {
|
|||||||
}
|
}
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PreviewRead is one preview as stored, with the parts of its key.
|
||||||
|
type PreviewRead struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Repository string `json:"repository"`
|
||||||
|
Head string `json:"head"`
|
||||||
|
Trunk string `json:"trunk"`
|
||||||
|
Preview json.RawMessage `json:"preview"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Previews are the previews the checker holds, each as stored. With a repository only that repository's; with a head (a
|
||||||
|
// prefix of at least 7 characters is enough) only previews of that head. A key that is no preview key is listed as unreadable.
|
||||||
|
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, error) {
|
||||||
|
if r.PreviewState.Keys == nil || r.PreviewState.Get == nil {
|
||||||
|
return nil, nil, fmt.Errorf("the previews state cannot be read here")
|
||||||
|
}
|
||||||
|
keys, err := r.PreviewState.Keys()
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
out := []PreviewRead{}
|
||||||
|
var bad []string
|
||||||
|
for _, k := range keys {
|
||||||
|
repo, h, trunk, err := delivery.ParsePreviewKey(k)
|
||||||
|
if err != nil {
|
||||||
|
bad = append(bad, k)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (repository != "" && repo != repository) || (head != "" && !strings.HasPrefix(h, head)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
raw, err := r.PreviewState.Get(k)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
if len(raw) > 0 {
|
||||||
|
out = append(out, PreviewRead{Key: k, Repository: repo, Head: h, Trunk: trunk, Preview: raw})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, bad, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunningStep is a step in flight on a build module.
|
||||||
|
type RunningStep struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Node string `json:"node"`
|
||||||
|
ID string `json:"id"`
|
||||||
|
Tool string `json:"tool"`
|
||||||
|
Since string `json:"since,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status is what holds the checker back and what runs now, read from the buckets: the checker's held and running steps live in
|
||||||
|
// its process, so they are shown as what the buckets say (the steps in flight on the build modules, the candidates without a
|
||||||
|
// verdict for their heads, and the delivery plans in their build phase), not as the process's own counters.
|
||||||
|
type Status struct {
|
||||||
|
Capacity map[string]int `json:"capacity"`
|
||||||
|
CapacityNote string `json:"capacity_note,omitempty"`
|
||||||
|
DefaultPerNode int `json:"default_per_node"`
|
||||||
|
PlansBuilding struct {
|
||||||
|
Readable bool `json:"readable"`
|
||||||
|
Keys []string `json:"keys"`
|
||||||
|
Holds bool `json:"holds"`
|
||||||
|
} `json:"plans_building"`
|
||||||
|
Running []RunningStep `json:"running_steps"`
|
||||||
|
RunningByNode map[string]int `json:"running_by_node"`
|
||||||
|
Unjudged []string `json:"unjudged_candidates"`
|
||||||
|
Notes []string `json:"notes,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status reads the capacity, the plan bucket, the build modules' steps and the candidates against their verdicts.
|
||||||
|
func (r Reader) Status() (Status, error) {
|
||||||
|
st := Status{DefaultPerNode: buildrun.DefaultCapacity, Capacity: map[string]int{}, RunningByNode: map[string]int{}}
|
||||||
|
if r.Capacity == nil {
|
||||||
|
st.CapacityNote = "the capacity setting cannot be read from here"
|
||||||
|
} else if setting, err := r.Capacity(); err != nil {
|
||||||
|
st.CapacityNote = "the capacity setting cannot be read: " + err.Error()
|
||||||
|
} else if st.Capacity, err = buildrun.ParseCapacity(setting); err != nil {
|
||||||
|
st.Capacity, st.CapacityNote = map[string]int{}, "the capacity setting is not valid: "+err.Error()
|
||||||
|
}
|
||||||
|
if r.PlansState.Keys == nil {
|
||||||
|
st.Notes = append(st.Notes, PlansBuildingBucket+" is not readable here (absent), so it is read as empty: no delivery holds the checker")
|
||||||
|
} else if keys, err := r.PlansState.Keys(); err != nil {
|
||||||
|
st.Notes = append(st.Notes, PlansBuildingBucket+" cannot be listed, so it is read as empty: "+err.Error())
|
||||||
|
} else {
|
||||||
|
sort.Strings(keys)
|
||||||
|
st.PlansBuilding.Readable, st.PlansBuilding.Keys, st.PlansBuilding.Holds = true, keys, len(keys) > 0
|
||||||
|
}
|
||||||
|
mods := make([]string, 0, len(r.StepStates))
|
||||||
|
for m := range r.StepStates {
|
||||||
|
mods = append(mods, m)
|
||||||
|
}
|
||||||
|
sort.Strings(mods)
|
||||||
|
st.Running = []RunningStep{}
|
||||||
|
for _, m := range mods {
|
||||||
|
sr := r.StepStates[m]
|
||||||
|
if sr.Keys == nil || sr.Get == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
keys, err := sr.Keys()
|
||||||
|
if err != nil {
|
||||||
|
st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
for _, k := range keys {
|
||||||
|
raw, err := sr.Get(k)
|
||||||
|
if err != nil || len(raw) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
step, err := delivery.DecodeStep(raw)
|
||||||
|
if err != nil || step.State != delivery.StepRunning {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool}
|
||||||
|
if !step.Started.IsZero() {
|
||||||
|
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
|
||||||
|
}
|
||||||
|
st.Running = append(st.Running, rs)
|
||||||
|
st.RunningByNode[step.Node]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
st.Unjudged = []string{}
|
||||||
|
if r.CandidateState.Keys != nil && r.CandidateState.Get != nil && r.VerdictState.Get != nil {
|
||||||
|
keys, err := r.CandidateState.Keys()
|
||||||
|
if err != nil {
|
||||||
|
return st, err
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
for _, k := range keys {
|
||||||
|
raw, err := r.CandidateState.Get(k)
|
||||||
|
if err != nil || len(raw) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c, err := delivery.DecodeCandidate(raw)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if v, err := r.VerdictState.Get(vkey); err == nil && len(v) == 0 {
|
||||||
|
st.Unjudged = append(st.Unjudged, c.Group)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
st.Notes = append(st.Notes, "the steps the checker holds waiting for a place are in its process and in no bucket: a candidate without a verdict while a plan key exists, or while its node runs as many steps as its capacity, is held")
|
||||||
|
return st, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,8 +3,10 @@ package check
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
@@ -102,3 +104,64 @@ func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { //
|
|||||||
t.Error("a digest nothing kept was answered")
|
t.Error("a digest nothing kept was answered")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1680 live acceptance: checker_previews shows the preview key of an open pull request, as stored, narrowed by repository and head.
|
||||||
|
func TestThePreviewsToolShowsTheStoredPreviewOfAPullRequest(t *testing.T) {
|
||||||
|
pv := delivery.Preview{Repository: repoA, Head: head1, Trunk: head2, Merge: "m1", Modules: map[string]delivery.PreviewModule{}}
|
||||||
|
data, err := delivery.EncodePreview(pv)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
key, _ := delivery.PreviewKey(repoA, head1, head2)
|
||||||
|
rd := Reader{PreviewState: fakeState{key: data, "junk": []byte("{}")}.read()}
|
||||||
|
got, bad, err := rd.Previews(repoA, head1[:8])
|
||||||
|
if err != nil || len(got) != 1 || got[0].Key != key || got[0].Head != head1 || got[0].Trunk != head2 || string(got[0].Preview) != string(data) {
|
||||||
|
t.Fatalf("previews = %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
if len(bad) != 1 || bad[0] != "junk" {
|
||||||
|
t.Errorf("unreadable = %v", bad)
|
||||||
|
}
|
||||||
|
if got, _, _ := rd.Previews("novox/other", ""); len(got) != 0 {
|
||||||
|
t.Errorf("another repository's = %+v", got)
|
||||||
|
}
|
||||||
|
if _, _, err := (Reader{}).Previews("", ""); err == nil {
|
||||||
|
t.Error("answered with no state to read")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1685 live acceptance: checker_status reads the capacity, whether a plan key holds the checker, the steps in flight by node and
|
||||||
|
// the candidates without a verdict.
|
||||||
|
func TestTheStatusToolReadsCapacityHoldsAndRunningSteps(t *testing.T) {
|
||||||
|
running, _ := delivery.EncodeStep(delivery.Step{ID: "abc1", Tool: "build_go_test", Node: "shanks", State: delivery.StepRunning, Started: time.Date(2026, 10, 22, 15, 0, 0, 0, time.UTC)})
|
||||||
|
passed, _ := delivery.EncodeStep(delivery.Step{ID: "abc2", Tool: "build_go_test", Node: "ace", State: delivery.StepPassed})
|
||||||
|
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||||
|
ckey, _ := delivery.CandidatesKey("feat/x")
|
||||||
|
rd := Reader{
|
||||||
|
Capacity: func() (string, error) { return "shanks=2,ace=0", nil },
|
||||||
|
PlansState: fakeState{"p1": []byte("{}")}.read(),
|
||||||
|
StepStates: map[string]StateRead{"mesh-build-go": fakeState{"shanks.abc1": running, "ace.abc2": passed}.read()},
|
||||||
|
CandidateState: fakeState{ckey: cand}.read(),
|
||||||
|
VerdictState: fakeState{}.read(),
|
||||||
|
}
|
||||||
|
st, err := rd.Status()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if st.Capacity["shanks"] != 2 || st.Capacity["ace"] != 0 || st.DefaultPerNode != 1 {
|
||||||
|
t.Errorf("capacity = %v default %d", st.Capacity, st.DefaultPerNode)
|
||||||
|
}
|
||||||
|
if !st.PlansBuilding.Holds || len(st.PlansBuilding.Keys) != 1 {
|
||||||
|
t.Errorf("plans = %+v", st.PlansBuilding)
|
||||||
|
}
|
||||||
|
if len(st.Running) != 1 || st.Running[0].Node != "shanks" || st.RunningByNode["shanks"] != 1 || st.RunningByNode["ace"] != 0 {
|
||||||
|
t.Errorf("running = %+v %v", st.Running, st.RunningByNode)
|
||||||
|
}
|
||||||
|
if len(st.Unjudged) != 1 || st.Unjudged[0] != "feat/x" {
|
||||||
|
t.Errorf("unjudged = %v", st.Unjudged)
|
||||||
|
}
|
||||||
|
// An absent plan bucket is read as empty and said; an unreadable capacity is said, not guessed.
|
||||||
|
st, err = Reader{Capacity: func() (string, error) { return "", os.ErrNotExist }}.Status()
|
||||||
|
if err != nil || st.PlansBuilding.Holds || st.CapacityNote == "" || len(st.Notes) == 0 {
|
||||||
|
t.Errorf("empty status = %+v, %v", st, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -118,7 +118,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
|||||||
t.Errorf("streams = %+v, want %+v", m.Streams, want)
|
t.Errorf("streams = %+v, want %+v", m.Streams, want)
|
||||||
}
|
}
|
||||||
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
|
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
|
||||||
if want := []string{"checker_kept", "checker_pull", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||||
}
|
}
|
||||||
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats", "bus"} {
|
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats", "bus"} {
|
||||||
|
|||||||
@@ -31,7 +31,9 @@
|
|||||||
],
|
],
|
||||||
"tools": [
|
"tools": [
|
||||||
"checker_kept",
|
"checker_kept",
|
||||||
|
"checker_previews",
|
||||||
"checker_pull",
|
"checker_pull",
|
||||||
|
"checker_status",
|
||||||
"checker_verdicts"
|
"checker_verdicts"
|
||||||
],
|
],
|
||||||
"invokes": [
|
"invokes": [
|
||||||
|
|||||||
@@ -35,18 +35,39 @@ func Tools(r view.Reader) []stdio.Tool {
|
|||||||
mods, err := r.Modules(only)
|
mods, err := r.Modules(only)
|
||||||
return map[string]any{"modules": mods}, err
|
return map[string]any{"modules": mods}, err
|
||||||
}},
|
}},
|
||||||
|
{Name: view.CandidateReadTool,
|
||||||
|
Description: "What the indexer stored for one candidate (the open pull requests of a branch) under candidates.<group>, as stored: " +
|
||||||
|
"the pull requests and heads, the base, and per pull request the touched modules with their new-form fingerprint (fp2) or why " +
|
||||||
|
"they have none (none2). Read-only.",
|
||||||
|
Input: map[string]any{"type": "object", "required": []string{"group"}, "properties": map[string]any{
|
||||||
|
"group": map[string]any{"type": "string", "description": "the candidate's group, the branch name its pull requests share"}}},
|
||||||
|
Run: func(args map[string]any) (any, error) {
|
||||||
|
group, _ := args["group"].(string)
|
||||||
|
return r.Candidate(group)
|
||||||
|
}},
|
||||||
|
{Name: view.CandidatesReadTool,
|
||||||
|
Description: "Every candidate the indexer holds, by group, each as stored under candidates.<group>; with module, only those with a " +
|
||||||
|
"pull request touching that module. Keys that cannot be read are listed as unreadable. Read-only.",
|
||||||
|
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||||
|
"module": map[string]any{"type": "string", "description": "only candidates touching this module"}}},
|
||||||
|
Run: func(args map[string]any) (any, error) {
|
||||||
|
module, _ := args["module"].(string)
|
||||||
|
c, bad, err := r.Candidates(module)
|
||||||
|
return map[string]any{"candidates": c, "unreadable": bad}, err
|
||||||
|
}},
|
||||||
{Name: view.BuilderBuildsReadTool,
|
{Name: view.BuilderBuildsReadTool,
|
||||||
Description: "The builder's build keys, each as stored in mesh-delivery-builder_builds: key, module, fingerprint (src2) and build " +
|
Description: "The builder's build keys, each as stored in mesh-delivery-builder_builds: key, module, fingerprint (src2) and build " +
|
||||||
"(state, by, step with the nodes it was built on, manifest, artifacts with their digests, toolchains, error). Narrow by module " +
|
"(state, by, step with the nodes it was built on, manifest, artifacts with their digests, toolchains, error). Narrow by module " +
|
||||||
"or state (queued, building, done, failed). Read-only.",
|
"or state (queued, building, done, failed); at most limit builds (default 50), newest revision first for a module, and the answer says when it was cut and lists keys that are no build key (unreadable). Without a module, give one to get its newest. Read-only.",
|
||||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||||
"module": map[string]any{"type": "string", "description": "only this module's builds"},
|
"module": map[string]any{"type": "string", "description": "only this module's builds"},
|
||||||
"state": map[string]any{"type": "string", "description": "only builds in this state"}}},
|
"state": map[string]any{"type": "string", "description": "only builds in this state"},
|
||||||
|
"limit": map[string]any{"type": "integer", "description": "at most this many builds (default 50), newest revision first"}}},
|
||||||
Run: func(args map[string]any) (any, error) {
|
Run: func(args map[string]any) (any, error) {
|
||||||
module, _ := args["module"].(string)
|
module, _ := args["module"].(string)
|
||||||
state, _ := args["state"].(string)
|
state, _ := args["state"].(string)
|
||||||
rows, err := r.BuilderBuilds(module, state)
|
limit, _ := args["limit"].(float64)
|
||||||
return map[string]any{"builds": rows}, err
|
return r.BuilderBuilds(module, state, int(limit))
|
||||||
}},
|
}},
|
||||||
{Name: view.BuilderBuildReadTool,
|
{Name: view.BuilderBuildReadTool,
|
||||||
Description: "One build of the builder as stored. Give the module and its fingerprint (fp2, src2:...), or the module alone for the build " +
|
Description: "One build of the builder as stored. Give the module and its fingerprint (fp2, src2:...), or the module alone for the build " +
|
||||||
@@ -65,7 +86,13 @@ func Tools(r view.Reader) []stdio.Tool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func read(state stdio.KeptState) view.StateRead {
|
func read(state stdio.KeptState) view.StateRead {
|
||||||
return view.StateRead{Keys: state.Keys, Get: func(key string) (json.RawMessage, error) {
|
return view.StateRead{Keys: state.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) {
|
||||||
|
e, err := state.Get(key)
|
||||||
|
if err != nil || e == nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
return e.Value, e.Revision, nil
|
||||||
|
}, Get: func(key string) (json.RawMessage, error) {
|
||||||
e, err := state.Get(key)
|
e, err := state.Get(key)
|
||||||
if err != nil || e == nil {
|
if err != nil || e == nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -75,7 +102,7 @@ func read(state stdio.KeptState) view.StateRead {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds"))}
|
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates"))}
|
||||||
if err := stdio.Serve("", Tools(r)); err != nil {
|
if err := stdio.Serve("", Tools(r)); err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err)
|
fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -117,13 +118,16 @@ func TestBuilderBuildsListsKeysAsStored(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
rows := out.(map[string]any)["builds"].([]view.BuilderRow)
|
rows := out.(view.BuildsAnswer).Builds
|
||||||
|
if u := out.(view.BuildsAnswer).Unreadable; len(u) != 1 || u[0] != "junk" {
|
||||||
|
t.Errorf("unreadable = %v, want [junk] (issue 1818)", u)
|
||||||
|
}
|
||||||
if len(rows) != 2 || rows[0].Key != keyA || rows[0].Module != "netcheck" || rows[0].Fingerprint != fp2A || string(rows[0].Build) != doneBuild || string(rows[1].Build) != failBuild {
|
if len(rows) != 2 || rows[0].Key != keyA || rows[0].Module != "netcheck" || rows[0].Fingerprint != fp2A || string(rows[0].Build) != doneBuild || string(rows[1].Build) != failBuild {
|
||||||
t.Errorf("rows = %+v", rows)
|
t.Errorf("rows = %+v", rows)
|
||||||
}
|
}
|
||||||
for args, want := range map[string]string{"state": keyB, "module": keyA} {
|
for args, want := range map[string]string{"state": keyB, "module": keyA} {
|
||||||
out, _ := run(t, r, view.BuilderBuildsReadTool, map[string]any{args: map[string]string{keyB: "failed", keyA: "netcheck"}[want]})
|
out, _ := run(t, r, view.BuilderBuildsReadTool, map[string]any{args: map[string]string{keyB: "failed", keyA: "netcheck"}[want]})
|
||||||
got := out.(map[string]any)["builds"].([]view.BuilderRow)
|
got := out.(view.BuildsAnswer).Builds
|
||||||
if len(got) != 1 || got[0].Key != want {
|
if len(got) != 1 || got[0].Key != want {
|
||||||
t.Errorf("%s filter = %+v, want %s", args, got, want)
|
t.Errorf("%s filter = %+v, want %s", args, got, want)
|
||||||
}
|
}
|
||||||
@@ -156,3 +160,107 @@ func TestBuilderBuildJoinsTheIndexersFingerprint(t *testing.T) {
|
|||||||
t.Errorf("an unbuilt fingerprint: %v", err)
|
t.Errorf("an unbuilt fingerprint: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// bigBuilds is a bucket of n keys over 100 modules that counts the values read.
|
||||||
|
type bigBuilds struct {
|
||||||
|
fake fake
|
||||||
|
reads *int
|
||||||
|
}
|
||||||
|
|
||||||
|
func newBig(n int) bigBuilds {
|
||||||
|
f := fake{}
|
||||||
|
for i := 0; i < n; i++ {
|
||||||
|
f[fmt.Sprintf("mod%02d.%064x", i%100, i)] = doneBuild
|
||||||
|
}
|
||||||
|
return bigBuilds{fake: f, reads: new(int)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b bigBuilds) reader() view.Reader {
|
||||||
|
r := b.fake.reader()
|
||||||
|
get := r.State.Get
|
||||||
|
r.Builds = view.StateRead{Keys: r.State.Keys, Get: func(k string) (json.RawMessage, error) { *b.reads++; return get(k) },
|
||||||
|
GetEntry: func(k string) (json.RawMessage, uint64, error) {
|
||||||
|
*b.reads++
|
||||||
|
raw, err := get(k)
|
||||||
|
var rev uint64
|
||||||
|
fmt.Sscanf(k[strings.Index(k, ".")+1:], "%x", &rev) // the fake's revision is the key's number
|
||||||
|
return raw, rev + 1, err
|
||||||
|
}}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1827/AC1: with 2,000 keys, a module's builds are found by key prefix, so only that module's values are read (20, not 2,000),
|
||||||
|
// newest revision first, within the limit, and the answer says it was cut.
|
||||||
|
func TestBuilderBuildsFiltersByPrefixBeforeAnyGet(t *testing.T) {
|
||||||
|
b := newBig(2000)
|
||||||
|
out, err := run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{"module": "mod07", "limit": float64(5)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a := out.(view.BuildsAnswer)
|
||||||
|
if *b.reads != 20 {
|
||||||
|
t.Errorf("read %d values for a module of 20 keys, want 20", *b.reads)
|
||||||
|
}
|
||||||
|
if len(a.Builds) != 5 || !a.Cut || !strings.Contains(a.Note, "5") {
|
||||||
|
t.Errorf("answer = %d rows, cut %v, note %q", len(a.Builds), a.Cut, a.Note)
|
||||||
|
}
|
||||||
|
for i := 1; i < len(a.Builds); i++ {
|
||||||
|
if a.Builds[i-1].Revision < a.Builds[i].Revision {
|
||||||
|
t.Errorf("not newest first: %d before %d", a.Builds[i-1].Revision, a.Builds[i].Revision)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if a.Builds[0].Revision != 1908 {
|
||||||
|
t.Errorf("newest revision = %d, want 1908", a.Builds[0].Revision)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1827/AC2: with no module the answer reads at most the limit's worth in key order (default 50), never the 2,000.
|
||||||
|
func TestBuilderBuildsWithoutAModuleIsBounded(t *testing.T) {
|
||||||
|
b := newBig(2000)
|
||||||
|
out, err := run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a := out.(view.BuildsAnswer)
|
||||||
|
if len(a.Builds) != view.DefaultBuildsLimit || !a.Cut || *b.reads > view.DefaultBuildsLimit {
|
||||||
|
t.Errorf("%d rows, cut %v, %d reads", len(a.Builds), a.Cut, *b.reads)
|
||||||
|
}
|
||||||
|
b = newBig(2000)
|
||||||
|
out, _ = run(t, b.reader(), view.BuilderBuildsReadTool, map[string]any{"state": "failed"})
|
||||||
|
if a := out.(view.BuildsAnswer); *b.reads != view.MaxBuildReads || !a.Cut || len(a.Builds) != 0 {
|
||||||
|
t.Errorf("a state matching nothing: %d reads (want the bound %d), cut %v", *b.reads, view.MaxBuildReads, a.Cut)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1818: fp2 together with commit is refused, naming both.
|
||||||
|
func TestBuilderBuildRefusesFP2WithCommit(t *testing.T) {
|
||||||
|
r := both(fake{keyA: doneBuild}, fake{"netcheck": `{"v":3,"fp2":"` + fp2A + `","commit":"c1","path":"."}`})
|
||||||
|
_, err := run(t, r, view.BuilderBuildReadTool, map[string]any{"module": "netcheck", "fp2": fp2A, "commit": "c1"})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "fp2") || !strings.Contains(err.Error(), "commit") || !strings.Contains(err.Error(), "not both") {
|
||||||
|
t.Errorf("err = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1681: indexer_candidates and indexer_candidate answer the stored value; a pull request's fp2 per touched module shows.
|
||||||
|
func TestCandidateToolsAnswerTheStoredValue(t *testing.T) {
|
||||||
|
cand := `{"v":1,"group":"feat/x","base":3,"heads":"h","pulls":[{"modules":["zsh"],"fps":{"zsh":{"fp2":"` + fp2B + `","manifest":"m"}}}]}`
|
||||||
|
other := `{"v":1,"group":"fix","base":3,"heads":"h","pulls":[{"modules":["netcheck"]}]}`
|
||||||
|
r := fake{"feat/x": cand, "fix": other, "bad": "{"}.reader()
|
||||||
|
r.CandidatesState = r.State
|
||||||
|
out, err := run(t, r, view.CandidateReadTool, map[string]any{"group": "feat/x"})
|
||||||
|
if err != nil || string(out.(json.RawMessage)) != cand || !strings.Contains(string(out.(json.RawMessage)), fp2B) {
|
||||||
|
t.Fatalf("candidate = %v, %v", out, err)
|
||||||
|
}
|
||||||
|
if _, err := run(t, r, view.CandidateReadTool, map[string]any{"group": "gone"}); err == nil || !strings.Contains(err.Error(), "no candidate gone") {
|
||||||
|
t.Errorf("err = %v", err)
|
||||||
|
}
|
||||||
|
out, err = run(t, r, view.CandidatesReadTool, map[string]any{})
|
||||||
|
m := out.(map[string]any)
|
||||||
|
if err != nil || len(m["candidates"].(map[string]json.RawMessage)) != 2 || len(m["unreadable"].([]string)) != 1 {
|
||||||
|
t.Errorf("all = %v, %v", m, err)
|
||||||
|
}
|
||||||
|
out, _ = run(t, r, view.CandidatesReadTool, map[string]any{"module": "zsh"})
|
||||||
|
if c := out.(map[string]any)["candidates"].(map[string]json.RawMessage); len(c) != 1 || c["feat/x"] == nil {
|
||||||
|
t.Errorf("by module = %v", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ package view
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -23,12 +24,16 @@ const (
|
|||||||
type StateRead struct {
|
type StateRead struct {
|
||||||
Keys func() ([]string, error)
|
Keys func() ([]string, error)
|
||||||
Get func(key string) (json.RawMessage, error)
|
Get func(key string) (json.RawMessage, error)
|
||||||
|
// GetEntry, when set, also gives the revision of the value; builder_builds orders by it.
|
||||||
|
GetEntry func(key string) (json.RawMessage, uint64, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reader answers the read tools from a modules state.
|
// Reader answers the read tools from a modules state.
|
||||||
type Reader struct {
|
type Reader struct {
|
||||||
State StateRead // the indexer's modules
|
State StateRead // the indexer's modules
|
||||||
Builds StateRead // the builder's builds
|
Builds StateRead // the builder's builds
|
||||||
|
|
||||||
|
CandidatesState StateRead // the indexer's candidates
|
||||||
}
|
}
|
||||||
|
|
||||||
// Module is the stored value of modules.<module>, exactly as the indexer wrote it.
|
// Module is the stored value of modules.<module>, exactly as the indexer wrote it.
|
||||||
@@ -84,6 +89,67 @@ func (r Reader) Modules(none2 bool) (map[string]json.RawMessage, error) {
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The read tools of the indexer's candidates (issue 1681): what the indexer stored under candidates.<group>, as stored, so the
|
||||||
|
// new-form fingerprint of each touched module of an open pull request (pulls[].fps) can be read live.
|
||||||
|
const (
|
||||||
|
CandidateReadTool = "indexer_candidate"
|
||||||
|
CandidatesReadTool = "indexer_candidates"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Candidate is the stored value of the candidates key of a group, exactly as the indexer wrote it.
|
||||||
|
func (r Reader) Candidate(group string) (json.RawMessage, error) {
|
||||||
|
key, err := delivery.CandidatesKey(strings.TrimSpace(group))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if r.CandidatesState.Get == nil {
|
||||||
|
return nil, fmt.Errorf("the candidates state cannot be read here")
|
||||||
|
}
|
||||||
|
raw, err := r.CandidatesState.Get(key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(raw) == 0 {
|
||||||
|
return nil, fmt.Errorf("no candidate %s in the candidates state", group)
|
||||||
|
}
|
||||||
|
return raw, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Candidates is every candidate, by group, each as stored; with a module only the candidates with a pull request touching it.
|
||||||
|
// A value that is not readable is listed under unreadable with its key, not skipped.
|
||||||
|
func (r Reader) Candidates(module string) (map[string]json.RawMessage, []string, error) {
|
||||||
|
if r.CandidatesState.Keys == nil || r.CandidatesState.Get == nil {
|
||||||
|
return nil, nil, fmt.Errorf("the candidates state cannot be read here")
|
||||||
|
}
|
||||||
|
keys, err := r.CandidatesState.Keys()
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
out := map[string]json.RawMessage{}
|
||||||
|
var unreadable []string
|
||||||
|
for _, k := range keys {
|
||||||
|
raw, err := r.CandidatesState.Get(k)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
if len(raw) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
group, perr := delivery.ParseCandidatesKey(k)
|
||||||
|
var c delivery.Candidate
|
||||||
|
if perr != nil || json.Unmarshal(raw, &c) != nil {
|
||||||
|
unreadable = append(unreadable, k)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if module != "" && !slices.ContainsFunc(c.Pulls, func(p delivery.CandidatePull) bool { return slices.Contains(p.Modules, module) }) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[group] = raw
|
||||||
|
}
|
||||||
|
return out, unreadable, nil
|
||||||
|
}
|
||||||
|
|
||||||
// The read tools of the builder's builds (novox/hq issue 1723): what the builder stored under `<module>.<fingerprint hex>`
|
// The read tools of the builder's builds (novox/hq issue 1723): what the builder stored under `<module>.<fingerprint hex>`
|
||||||
// in mesh-delivery-builder_builds, answered as stored, so its states, nodes (in the step) and digests can be checked live.
|
// in mesh-delivery-builder_builds, answered as stored, so its states, nodes (in the step) and digests can be checked live.
|
||||||
const (
|
const (
|
||||||
@@ -91,48 +157,100 @@ const (
|
|||||||
BuilderBuildsReadTool = "builder_builds"
|
BuilderBuildsReadTool = "builder_builds"
|
||||||
)
|
)
|
||||||
|
|
||||||
// BuilderRow is one key of the builder's builds: the key, the module and the fingerprint it parses to, and the stored value
|
// BuilderRow is one key of the builder's builds: the key, the module and the fingerprint it parses to, the revision of its
|
||||||
// as the builder wrote it (state, by, step, manifest, artifacts with digests, toolchains, error).
|
// stored value (when the read gives one) and the stored value as the builder wrote it.
|
||||||
type BuilderRow struct {
|
type BuilderRow struct {
|
||||||
Key string `json:"key"`
|
Key string `json:"key"`
|
||||||
Module string `json:"module"`
|
Module string `json:"module"`
|
||||||
Fingerprint string `json:"fingerprint"`
|
Fingerprint string `json:"fingerprint"`
|
||||||
|
Revision uint64 `json:"revision,omitempty"`
|
||||||
Build json.RawMessage `json:"build"`
|
Build json.RawMessage `json:"build"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The bounds of builder_builds (issue 1827): the rows it returns when no limit is given, and the values it reads in one call.
|
||||||
|
// Reading a value is a bus round trip, so the answer must not read the whole bucket (it timed out at 30 s on the live one).
|
||||||
|
const (
|
||||||
|
DefaultBuildsLimit = 50
|
||||||
|
MaxBuildReads = 200
|
||||||
|
)
|
||||||
|
|
||||||
|
// BuildsAnswer is the answer of builder_builds: the rows, newest revision first, whether the answer was cut and why, and
|
||||||
|
// the keys that are no build key (Unreadable), which are listed rather than silently skipped (issue 1818).
|
||||||
|
type BuildsAnswer struct {
|
||||||
|
Builds []BuilderRow `json:"builds"`
|
||||||
|
Cut bool `json:"cut,omitempty"`
|
||||||
|
Note string `json:"note,omitempty"`
|
||||||
|
Unreadable []string `json:"unreadable,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r Reader) get(key string) (json.RawMessage, uint64, error) {
|
||||||
|
if r.Builds.GetEntry != nil {
|
||||||
|
return r.Builds.GetEntry(key)
|
||||||
|
}
|
||||||
|
raw, err := r.Builds.Get(key)
|
||||||
|
return raw, 0, err
|
||||||
|
}
|
||||||
|
|
||||||
func (r Reader) builderRow(key string) (BuilderRow, bool, error) {
|
func (r Reader) builderRow(key string) (BuilderRow, bool, error) {
|
||||||
module, fp, err := delivery.ParseBuilderBuildKey(key)
|
module, fp, err := delivery.ParseBuilderBuildKey(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return BuilderRow{}, false, nil // a key that is no build key is not a build
|
return BuilderRow{}, false, nil // a key that is no build key is not a build
|
||||||
}
|
}
|
||||||
raw, err := r.Builds.Get(key)
|
raw, rev, err := r.get(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return BuilderRow{}, false, err
|
return BuilderRow{}, false, err
|
||||||
}
|
}
|
||||||
if len(raw) == 0 {
|
if len(raw) == 0 {
|
||||||
return BuilderRow{}, false, nil
|
return BuilderRow{}, false, nil
|
||||||
}
|
}
|
||||||
return BuilderRow{Key: key, Module: module, Fingerprint: fp, Build: raw}, true, nil
|
return BuilderRow{Key: key, Module: module, Fingerprint: fp, Revision: rev, Build: raw}, true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuilderBuilds lists the builder's keys, each as stored, sorted by key; module and state, when given, keep only the builds
|
// BuilderBuilds lists the builder's keys, each as stored, newest revision first. With a module it considers only the keys
|
||||||
// of that module or in that state.
|
// that begin `<module>.`, before any value is read; with a state it keeps the builds in that state. It returns at most limit
|
||||||
func (r Reader) BuilderBuilds(module, state string) ([]BuilderRow, error) {
|
// rows (DefaultBuildsLimit when limit is not above zero) and reads at most MaxBuildReads values, and says when it cut the
|
||||||
|
// answer. Keys that are no build key are listed as unreadable. Without a module the keys are walked in key order, so the
|
||||||
|
// answer is the first rows found, not the newest of the bucket: give a module to get its newest.
|
||||||
|
func (r Reader) BuilderBuilds(module, state string, limit int) (BuildsAnswer, error) {
|
||||||
if r.Builds.Keys == nil || r.Builds.Get == nil {
|
if r.Builds.Keys == nil || r.Builds.Get == nil {
|
||||||
return nil, fmt.Errorf("the builder's builds cannot be read here")
|
return BuildsAnswer{}, fmt.Errorf("the builder's builds cannot be read here")
|
||||||
|
}
|
||||||
|
if limit <= 0 {
|
||||||
|
limit = DefaultBuildsLimit
|
||||||
}
|
}
|
||||||
keys, err := r.Builds.Keys()
|
keys, err := r.Builds.Keys()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return BuildsAnswer{}, err
|
||||||
}
|
}
|
||||||
sort.Strings(keys)
|
sort.Strings(keys)
|
||||||
rows := []BuilderRow{}
|
prefix := ""
|
||||||
|
if module = strings.TrimSpace(module); module != "" {
|
||||||
|
prefix = module + "."
|
||||||
|
}
|
||||||
|
out := BuildsAnswer{Builds: []BuilderRow{}}
|
||||||
|
var candidates []string
|
||||||
for _, k := range keys {
|
for _, k := range keys {
|
||||||
|
if prefix != "" && !strings.HasPrefix(k, prefix) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, _, err := delivery.ParseBuilderBuildKey(k); err != nil {
|
||||||
|
out.Unreadable = append(out.Unreadable, k)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
candidates = append(candidates, k)
|
||||||
|
}
|
||||||
|
reads := 0
|
||||||
|
for _, k := range candidates {
|
||||||
|
if reads >= MaxBuildReads {
|
||||||
|
out.Cut, out.Note = true, fmt.Sprintf("read %d values and stopped; give a module (and a state) to narrow the answer", MaxBuildReads)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
reads++
|
||||||
row, ok, err := r.builderRow(k)
|
row, ok, err := r.builderRow(k)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return BuildsAnswer{}, err
|
||||||
}
|
}
|
||||||
if !ok || (module != "" && row.Module != module) {
|
if !ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if state != "" {
|
if state != "" {
|
||||||
@@ -143,9 +261,26 @@ func (r Reader) BuilderBuilds(module, state string) ([]BuilderRow, error) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
rows = append(rows, row)
|
out.Builds = append(out.Builds, row)
|
||||||
|
// Without a module the walk is in key order and stops at the limit; with one, all its (few) keys are read to order them.
|
||||||
|
if prefix == "" && len(out.Builds) >= limit {
|
||||||
|
if reads < len(candidates) {
|
||||||
|
out.Cut, out.Note = true, fmt.Sprintf("stopped at the limit of %d builds in key order, not newest first; give a module for its newest", limit)
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return rows, nil
|
sort.SliceStable(out.Builds, func(i, j int) bool {
|
||||||
|
if out.Builds[i].Revision != out.Builds[j].Revision {
|
||||||
|
return out.Builds[i].Revision > out.Builds[j].Revision
|
||||||
|
}
|
||||||
|
return out.Builds[i].Key < out.Builds[j].Key
|
||||||
|
})
|
||||||
|
if len(out.Builds) > limit {
|
||||||
|
out.Builds = out.Builds[:limit]
|
||||||
|
out.Cut, out.Note = true, fmt.Sprintf("cut to the newest %d builds of %s", limit, module)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuilderBuild is the build of module at the new-form fingerprint fp2; with an empty fp2 it is the build at the fingerprint
|
// BuilderBuild is the build of module at the new-form fingerprint fp2; with an empty fp2 it is the build at the fingerprint
|
||||||
@@ -156,6 +291,9 @@ func (r Reader) BuilderBuild(module, fp2, commit string) (BuilderRow, error) {
|
|||||||
if r.Builds.Get == nil {
|
if r.Builds.Get == nil {
|
||||||
return BuilderRow{}, fmt.Errorf("the builder's builds cannot be read here")
|
return BuilderRow{}, fmt.Errorf("the builder's builds cannot be read here")
|
||||||
}
|
}
|
||||||
|
if fp2 != "" && commit != "" {
|
||||||
|
return BuilderRow{}, fmt.Errorf("give fp2 or commit, not both: fp2 %s names a build directly, commit %s asks for the build at the indexer's fingerprint of that commit", fp2, commit)
|
||||||
|
}
|
||||||
if fp2 == "" {
|
if fp2 == "" {
|
||||||
raw, err := r.Module(module)
|
raw, err := r.Module(module)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -95,7 +95,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
|
|||||||
}
|
}
|
||||||
// The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer
|
// The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer
|
||||||
// itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads.
|
// itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads.
|
||||||
if want := []string{"indexer_module", "indexer_modules", "builder_build", "builder_builds"}; !reflect.DeepEqual(m.Tools, want) {
|
if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds"}; !reflect.DeepEqual(m.Tools, want) {
|
||||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||||
}
|
}
|
||||||
for _, a := range m.Build.Artifacts {
|
for _, a := range m.Build.Artifacts {
|
||||||
|
|||||||
@@ -6,6 +6,8 @@
|
|||||||
"tools": [
|
"tools": [
|
||||||
"indexer_module",
|
"indexer_module",
|
||||||
"indexer_modules",
|
"indexer_modules",
|
||||||
|
"indexer_candidate",
|
||||||
|
"indexer_candidates",
|
||||||
"builder_build",
|
"builder_build",
|
||||||
"builder_builds"
|
"builder_builds"
|
||||||
],
|
],
|
||||||
|
|||||||
Reference in New Issue
Block a user