331 lines
11 KiB
Go
331 lines
11 KiB
Go
package view
|
|
|
|
// The read tools of the indexer's modules (novox/hq issue 1562): what the indexer stored under modules.<m>, answered as
|
|
// stored. The new-form fingerprint (FP2, None2, Inputs2), the reason it has none, and the path and toolchains it was computed
|
|
// with are only there. The view reads the bucket through its reads; the reads write nothing.
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
)
|
|
|
|
// The read tools, named as the module is: indexer_ and the verb.
|
|
const (
|
|
ModuleReadTool = "indexer_module"
|
|
ModulesReadTool = "indexer_modules"
|
|
)
|
|
|
|
// StateRead reads the indexer's modules state, as the view's reads grant it: its keys and a key's stored value (nil for a key with none).
|
|
type StateRead struct {
|
|
Keys func() ([]string, error)
|
|
Get func(key string) (json.RawMessage, error)
|
|
// GetEntry, when set, also gives the revision of the value; builder_builds orders by it.
|
|
GetEntry func(key string) (json.RawMessage, uint64, error)
|
|
}
|
|
|
|
// Reader answers the read tools from a modules state.
|
|
type Reader struct {
|
|
State StateRead // the indexer's modules
|
|
Builds StateRead // the builder's builds
|
|
|
|
CandidatesState StateRead // the indexer's candidates
|
|
}
|
|
|
|
// Module is the stored value of modules.<module>, exactly as the indexer wrote it.
|
|
func (r Reader) Module(module string) (json.RawMessage, error) {
|
|
module = strings.TrimSpace(module)
|
|
key, err := delivery.ModulesKey(module)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if r.State.Get == nil {
|
|
return nil, fmt.Errorf("the modules state cannot be read here")
|
|
}
|
|
raw, err := r.State.Get(key)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(raw) == 0 {
|
|
return nil, fmt.Errorf("no module %s in the modules state", module)
|
|
}
|
|
return raw, nil
|
|
}
|
|
|
|
// Modules is every key of the modules state with its stored value; with none2 only the modules that have no new-form
|
|
// fingerprint and say why (a None2 reason), which is what the fingerprint's check reads live.
|
|
func (r Reader) Modules(none2 bool) (map[string]json.RawMessage, error) {
|
|
if r.State.Keys == nil || r.State.Get == nil {
|
|
return nil, fmt.Errorf("the modules state cannot be read here")
|
|
}
|
|
keys, err := r.State.Keys()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sort.Strings(keys)
|
|
out := map[string]json.RawMessage{}
|
|
for _, k := range keys {
|
|
raw, err := r.State.Get(k)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(raw) == 0 {
|
|
continue
|
|
}
|
|
if none2 {
|
|
var m struct {
|
|
None2 string `json:"none2"`
|
|
}
|
|
if json.Unmarshal(raw, &m) != nil || m.None2 == "" {
|
|
continue
|
|
}
|
|
}
|
|
out[k] = raw
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// The read tools of the indexer's candidates (issue 1681): what the indexer stored under candidates.<group>, as stored, so the
|
|
// new-form fingerprint of each touched module of an open pull request (pulls[].fps) can be read live.
|
|
const (
|
|
CandidateReadTool = "indexer_candidate"
|
|
CandidatesReadTool = "indexer_candidates"
|
|
)
|
|
|
|
// Candidate is the stored value of the candidates key of a group, exactly as the indexer wrote it.
|
|
func (r Reader) Candidate(group string) (json.RawMessage, error) {
|
|
key, err := delivery.CandidatesKey(strings.TrimSpace(group))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if r.CandidatesState.Get == nil {
|
|
return nil, fmt.Errorf("the candidates state cannot be read here")
|
|
}
|
|
raw, err := r.CandidatesState.Get(key)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(raw) == 0 {
|
|
return nil, fmt.Errorf("no candidate %s in the candidates state", group)
|
|
}
|
|
return raw, nil
|
|
}
|
|
|
|
// Candidates is every candidate, by group, each as stored; with a module only the candidates with a pull request touching it.
|
|
// A value that is not readable is listed under unreadable with its key, not skipped.
|
|
func (r Reader) Candidates(module string) (map[string]json.RawMessage, []string, error) {
|
|
if r.CandidatesState.Keys == nil || r.CandidatesState.Get == nil {
|
|
return nil, nil, fmt.Errorf("the candidates state cannot be read here")
|
|
}
|
|
keys, err := r.CandidatesState.Keys()
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
sort.Strings(keys)
|
|
out := map[string]json.RawMessage{}
|
|
var unreadable []string
|
|
for _, k := range keys {
|
|
raw, err := r.CandidatesState.Get(k)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if len(raw) == 0 {
|
|
continue
|
|
}
|
|
group, perr := delivery.ParseCandidatesKey(k)
|
|
var c delivery.Candidate
|
|
if perr != nil || json.Unmarshal(raw, &c) != nil {
|
|
unreadable = append(unreadable, k)
|
|
continue
|
|
}
|
|
if module != "" && !slices.ContainsFunc(c.Pulls, func(p delivery.CandidatePull) bool { return slices.Contains(p.Modules, module) }) {
|
|
continue
|
|
}
|
|
out[group] = raw
|
|
}
|
|
return out, unreadable, nil
|
|
}
|
|
|
|
// The read tools of the builder's builds (novox/hq issue 1723): what the builder stored under `<module>.<fingerprint hex>`
|
|
// in mesh-delivery-builder_builds, answered as stored, so its states, nodes (in the step) and digests can be checked live.
|
|
const (
|
|
BuilderBuildReadTool = "builder_build"
|
|
BuilderBuildsReadTool = "builder_builds"
|
|
)
|
|
|
|
// BuilderRow is one key of the builder's builds: the key, the module and the fingerprint it parses to, the revision of its
|
|
// stored value (when the read gives one) and the stored value as the builder wrote it.
|
|
type BuilderRow struct {
|
|
Key string `json:"key"`
|
|
Module string `json:"module"`
|
|
Fingerprint string `json:"fingerprint"`
|
|
Revision uint64 `json:"revision,omitempty"`
|
|
Build json.RawMessage `json:"build"`
|
|
}
|
|
|
|
// The bounds of builder_builds (issue 1827): the rows it returns when no limit is given, and the values it reads in one call.
|
|
// Reading a value is a bus round trip, so the answer must not read the whole bucket (it timed out at 30 s on the live one).
|
|
const (
|
|
DefaultBuildsLimit = 50
|
|
MaxBuildReads = 200
|
|
)
|
|
|
|
// BuildsAnswer is the answer of builder_builds: the rows, newest revision first, whether the answer was cut and why, and
|
|
// the keys that are no build key (Unreadable), which are listed rather than silently skipped (issue 1818).
|
|
type BuildsAnswer struct {
|
|
Builds []BuilderRow `json:"builds"`
|
|
Cut bool `json:"cut,omitempty"`
|
|
Note string `json:"note,omitempty"`
|
|
Unreadable []string `json:"unreadable,omitempty"`
|
|
}
|
|
|
|
func (r Reader) get(key string) (json.RawMessage, uint64, error) {
|
|
if r.Builds.GetEntry != nil {
|
|
return r.Builds.GetEntry(key)
|
|
}
|
|
raw, err := r.Builds.Get(key)
|
|
return raw, 0, err
|
|
}
|
|
|
|
func (r Reader) builderRow(key string) (BuilderRow, bool, error) {
|
|
module, fp, err := delivery.ParseBuilderBuildKey(key)
|
|
if err != nil {
|
|
return BuilderRow{}, false, nil // a key that is no build key is not a build
|
|
}
|
|
raw, rev, err := r.get(key)
|
|
if err != nil {
|
|
return BuilderRow{}, false, err
|
|
}
|
|
if len(raw) == 0 {
|
|
return BuilderRow{}, false, nil
|
|
}
|
|
return BuilderRow{Key: key, Module: module, Fingerprint: fp, Revision: rev, Build: raw}, true, nil
|
|
}
|
|
|
|
// BuilderBuilds lists the builder's keys, each as stored, newest revision first. With a module it considers only the keys
|
|
// that begin `<module>.`, before any value is read; with a state it keeps the builds in that state. It returns at most limit
|
|
// rows (DefaultBuildsLimit when limit is not above zero) and reads at most MaxBuildReads values, and says when it cut the
|
|
// answer. Keys that are no build key are listed as unreadable. Without a module the keys are walked in key order, so the
|
|
// answer is the first rows found, not the newest of the bucket: give a module to get its newest.
|
|
func (r Reader) BuilderBuilds(module, state string, limit int) (BuildsAnswer, error) {
|
|
if r.Builds.Keys == nil || r.Builds.Get == nil {
|
|
return BuildsAnswer{}, fmt.Errorf("the builder's builds cannot be read here")
|
|
}
|
|
if limit <= 0 {
|
|
limit = DefaultBuildsLimit
|
|
}
|
|
keys, err := r.Builds.Keys()
|
|
if err != nil {
|
|
return BuildsAnswer{}, err
|
|
}
|
|
sort.Strings(keys)
|
|
prefix := ""
|
|
if module = strings.TrimSpace(module); module != "" {
|
|
prefix = module + "."
|
|
}
|
|
out := BuildsAnswer{Builds: []BuilderRow{}}
|
|
var candidates []string
|
|
for _, k := range keys {
|
|
if prefix != "" && !strings.HasPrefix(k, prefix) {
|
|
continue
|
|
}
|
|
if _, _, err := delivery.ParseBuilderBuildKey(k); err != nil {
|
|
out.Unreadable = append(out.Unreadable, k)
|
|
continue
|
|
}
|
|
candidates = append(candidates, k)
|
|
}
|
|
reads := 0
|
|
for _, k := range candidates {
|
|
if reads >= MaxBuildReads {
|
|
out.Cut, out.Note = true, fmt.Sprintf("read %d values and stopped; give a module (and a state) to narrow the answer", MaxBuildReads)
|
|
break
|
|
}
|
|
reads++
|
|
row, ok, err := r.builderRow(k)
|
|
if err != nil {
|
|
return BuildsAnswer{}, err
|
|
}
|
|
if !ok {
|
|
continue
|
|
}
|
|
if state != "" {
|
|
var b struct {
|
|
State string `json:"state"`
|
|
}
|
|
if json.Unmarshal(row.Build, &b) != nil || b.State != state {
|
|
continue
|
|
}
|
|
}
|
|
out.Builds = append(out.Builds, row)
|
|
// Without a module the walk is in key order and stops at the limit; with one, all its (few) keys are read to order them.
|
|
if prefix == "" && len(out.Builds) >= limit {
|
|
if reads < len(candidates) {
|
|
out.Cut, out.Note = true, fmt.Sprintf("stopped at the limit of %d builds in key order, not newest first; give a module for its newest", limit)
|
|
}
|
|
break
|
|
}
|
|
}
|
|
sort.SliceStable(out.Builds, func(i, j int) bool {
|
|
if out.Builds[i].Revision != out.Builds[j].Revision {
|
|
return out.Builds[i].Revision > out.Builds[j].Revision
|
|
}
|
|
return out.Builds[i].Key < out.Builds[j].Key
|
|
})
|
|
if len(out.Builds) > limit {
|
|
out.Builds = out.Builds[:limit]
|
|
out.Cut, out.Note = true, fmt.Sprintf("cut to the newest %d builds of %s", limit, module)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// BuilderBuild is the build of module at the new-form fingerprint fp2; with an empty fp2 it is the build at the fingerprint
|
|
// the indexer holds for the module now (its fp2), joined through modules.<module>; commit, when given, must be the commit
|
|
// the indexer holds that fp2 for, or the answer says which commit it holds (the fingerprint of another commit is not guessed).
|
|
func (r Reader) BuilderBuild(module, fp2, commit string) (BuilderRow, error) {
|
|
module = strings.TrimSpace(module)
|
|
if r.Builds.Get == nil {
|
|
return BuilderRow{}, fmt.Errorf("the builder's builds cannot be read here")
|
|
}
|
|
if fp2 != "" && commit != "" {
|
|
return BuilderRow{}, fmt.Errorf("give fp2 or commit, not both: fp2 %s names a build directly, commit %s asks for the build at the indexer's fingerprint of that commit", fp2, commit)
|
|
}
|
|
if fp2 == "" {
|
|
raw, err := r.Module(module)
|
|
if err != nil {
|
|
return BuilderRow{}, err
|
|
}
|
|
var m struct {
|
|
FP2 string `json:"fp2"`
|
|
None2 string `json:"none2"`
|
|
Commit string `json:"commit"`
|
|
}
|
|
if err := json.Unmarshal(raw, &m); err != nil {
|
|
return BuilderRow{}, fmt.Errorf("the indexer's value of %s is not readable: %w", module, err)
|
|
}
|
|
if commit != "" && m.Commit != commit {
|
|
return BuilderRow{}, fmt.Errorf("the indexer holds %s at commit %s, not %s, so its fingerprint is not the one of that commit", module, m.Commit, commit)
|
|
}
|
|
if m.FP2 == "" {
|
|
return BuilderRow{}, fmt.Errorf("the indexer holds no new-form fingerprint of %s: %s", module, m.None2)
|
|
}
|
|
fp2 = m.FP2
|
|
}
|
|
key, err := delivery.BuilderBuildKey(module, fp2)
|
|
if err != nil {
|
|
return BuilderRow{}, err
|
|
}
|
|
row, ok, err := r.builderRow(key)
|
|
if err != nil {
|
|
return BuilderRow{}, err
|
|
}
|
|
if !ok {
|
|
return BuilderRow{}, fmt.Errorf("the builder holds no build %s", key)
|
|
}
|
|
return row, nil
|
|
}
|