The delivery repository's contracts and SDK live under modules/ as mesh-delivery-contracts and mesh-delivery-sdk, their Go module paths renamed to match the folders (the TypeScript names already did), and every importer in mesh-delivery, mesh-catalog, mesh-host, mesh-tools and mesh-sdk follows in the same group: a design as it should be, one path per module, checked by the group's check building every importer. Builds: design 47 G §427, §88 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
844 lines
37 KiB
Go
844 lines
37 KiB
Go
package check
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"mesh-delivery-checker/rules"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliverycheckerclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
|
)
|
|
|
|
// RunnerWorkspace is the runner the Go build module's digest and test steps are asked with when the merge changes the build
|
|
// module itself: the installed module compiles the candidate's program from the merged tree and runs the step as it, shut
|
|
// off (ADR 0535 §1).
|
|
const RunnerWorkspace = "workspace"
|
|
|
|
// buildGoDir is the Go build module's directory in the catalogue: a merge changing a path under it is judged by its
|
|
// candidate (ADR 0535 §1). The other build modules' own directories are judged by the installed module until its part of issue 1898.
|
|
func buildGoDir(goModule string) string { return "modules/" + goModule + "/" }
|
|
|
|
// changesBuildGo says whether a merge of the repository with these changed paths changes the Go build module.
|
|
func changesBuildGo(repository, goModule string, changed []string) bool {
|
|
if repository != catalogueRepository || goModule == "" {
|
|
return false
|
|
}
|
|
for _, p := range changed {
|
|
if strings.HasPrefix(p, buildGoDir(goModule)) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// The words of the line on the program that judged a change to the Go build module (ADR 0535 §2, §5).
|
|
func judgedByOldInstalled(module string) string {
|
|
return "judged by the installed " + module + ": it does not run a candidate yet"
|
|
}
|
|
func judgedByInstalled(module string) string { return "judged by the installed " + module }
|
|
func judgedByCandidate(module string) string { return "judged by the candidate's " + module + " at " }
|
|
|
|
// Asks puts a check's ask on its build seat's stream and waits for its step (buildrun.Asker; ADR 0525 §1, §6): the
|
|
// checker chooses no node, starts no step and holds none; whichever build node pulls the ask runs the check.
|
|
type Asks interface {
|
|
Put(ctx context.Context, a buildstep.StepAsk) error
|
|
Wait(ctx context.Context, a buildstep.StepAsk) (buildstep.Step, error)
|
|
}
|
|
|
|
// ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet).
|
|
type ToolchainEntries interface {
|
|
Entries(language string) []buildstep.NodeToolchain
|
|
}
|
|
|
|
// BuildModules says which build modules judge a repository: the Go one always, the Node one when the repository holds a
|
|
// TypeScript module.
|
|
type BuildModules interface {
|
|
BuildModules(repository string) []string
|
|
}
|
|
|
|
// Examiner judges a case through the build seats (ADR 0484 clauses 3, 4, 10, 13; ADR 0525 §1, §3): for each pull request
|
|
// and build module it puts one check ask whose job fetches the head merged onto its trunk head, then runs the unit level
|
|
// and the integration level, taking the kept verdicts of unchanged packages and running the change's own tests first;
|
|
// every level and member runs to its end also after a failure, so the verdict names every failure (the operator,
|
|
// 2026-11-02). It judges the step's answer as it judged the tools' answers.
|
|
type Examiner struct {
|
|
Asks Asks
|
|
// DeclineBound is how long a placed ask may go on being declined, from its first decline, before the check fails
|
|
// saying no node with its need took it (issue 890, ADR 0552); DefaultDeclineBound when zero.
|
|
DeclineBound time.Duration
|
|
// Toolchains are the nodes' toolchains: a kept verdict names the toolchain the nodes agree on.
|
|
Toolchains ToolchainEntries
|
|
Kept Kept
|
|
// Previews is the previews bucket; nil writes none.
|
|
Previews KV
|
|
Modules BuildModules
|
|
// Seats finds the build modules from the seat list and the holders of seats (design 47 G A-82).
|
|
Seats *buildrun.BuildSeats
|
|
// Runs says which commit the mesh runs of a module (the indexer's modules); nil knows none.
|
|
Runs interface{ CommitOf(module string) string }
|
|
// Forge reads the forge's files for the siblings a repository's tests read; nil reads none.
|
|
Forge Forge
|
|
// Published names the latest published version of every mesh package a module of the mesh uses, by package, from the
|
|
// indexer's modules (novox/hq ADR 0573 §2): a check builds against those, or against a member's head for a package
|
|
// its group publishes (§4), whatever go.mod names. Nil names none, and go.mod's versions are built.
|
|
Published interface {
|
|
LatestPackages() buildstep.PackageVersions
|
|
}
|
|
// KnownPackages names the packages the indexer's packages state knows (its series keys, meshcatalog.PackagesByPackageSeriesValue), by the
|
|
// name in its own ecosystem, each with the `<module>/<artifact>` it is built from: the names a manifest's `packages` is
|
|
// judged against (manifests_judge.go, design 32). Nil, or PackageNames answering nil while the state is not read yet,
|
|
// hands the judge no names, and it judges none; an empty map is a state read and empty, against which every name fails.
|
|
KnownPackages interface {
|
|
PackageNames() map[string]string
|
|
}
|
|
// Builds says how the build of a fingerprint the candidate carries stands (the builder's builds bucket); nil says nothing
|
|
// of the builds.
|
|
Builds BuildStates
|
|
// BuildEnds waits on the builder's builds of the candidate before the check composes (ADR 0603 §10); nil composes
|
|
// without waiting, said in the composition's warnings.
|
|
BuildEnds BuildEnds
|
|
// BuildWait is how long the check waits for those builds (the setting build-wait-bound); DefaultBuildWait when 0.
|
|
BuildWait time.Duration
|
|
// Announce says a preview was written under the key (EventPreviewWritten); nil says nothing.
|
|
Announce Announcer
|
|
// Dirs names a module's directory in its repository, from the indexer's modules, so a manifest is read at a member's
|
|
// head (tagwanted.go); nil, or a module it does not know, tries modules/<module> and the repository's root.
|
|
Dirs interface {
|
|
ModuleDir(module string) (string, bool)
|
|
}
|
|
// Packages names, of a repository, the paths (repository-relative) its TypeScript package artifacts are built from, from
|
|
// the indexer's modules (meshcatalog.ModulesByModuleValue.ArtifactPaths, lead decision L8); nil serves no group package.
|
|
Packages interface {
|
|
PackagePaths(repository string) []string
|
|
}
|
|
// Trunks are the trunk heads of every repository, for the siblings a repository's tests read.
|
|
Trunks interface{ Heads() map[string]string }
|
|
// Manifests are the indexer's manifest digests by module: the catalogue a change's manifests are judged in by the
|
|
// indexer's judge (manifests_judge.go, novox/hq ADR 0578 §2); nil judges no manifest.
|
|
Manifests TrunkManifests
|
|
// SeatDefinitions reads the indexer's seat definitions of a pull request (seatdefs.go, design 26); nil reads none.
|
|
SeatDefinitions SeatDefinitions
|
|
// SeatList gives the seat list the checker's reader holds from the seat mesh-catalog (verb seat-definitions): the base the
|
|
// checking program lays the heads' definitions over (programs.go, novox/hq ADR 0636 §1, §4); nil gives it none.
|
|
SeatList SeatList
|
|
// Renames asks the seat mesh-config's verb renames for a change that brings or drops a `renamed-from` (renames.go,
|
|
// design 32); nil fails such a change, saying it cannot ask.
|
|
Renames Renames
|
|
// Read reads a kept manifest by its digest: the change's module graph is worked out from them (graph_edges.go, novox/hq
|
|
// ADR 0603 §2); nil works out no graph.
|
|
Read ManifestReader
|
|
// Serving says, from what runs, whether a running healthy build serves a seat verb or module tool (novox/hq ADR 0614
|
|
// §7): the node-engines' haves, the builder's build of each, its manifest (graph.Served). Nil reads nothing, and no
|
|
// changed build module is named for a verb it invokes.
|
|
Serving interface {
|
|
Served(ctx context.Context) (func(invoke string) bool, error)
|
|
}
|
|
// Predefined reads the catalogue's predefined keys the indexer keeps (manifests_judge.go); nil reads none.
|
|
Predefined PredefinedList
|
|
// Compositions makes a check's composition (ADR 0593 §7): the engines' answers (EngineCompositions); nil leaves the
|
|
// composition not run.
|
|
Compositions CompositionReader
|
|
// Nodes names every node with a record, read from each `<node>.node` of the node-engine's have (novox/hq ADR 0567
|
|
// §12): every check ask carries them (CheckJob.NodeNames) for the names check, so no repository lists them. A check
|
|
// whose names cannot be read, or that reads none, is not asked and fails saying so; nil hands no names.
|
|
Nodes interface {
|
|
Names(ctx context.Context) ([]string, error)
|
|
}
|
|
// Candidates answers the indexer's candidate of a try's group once it wrote it from the try's previews, the manifests
|
|
// and fingerprints a group try's manifest validator and composition judge (issue 1921), and of a pull request's check
|
|
// that began before the indexer read its merge preview; nil leaves such a case with no manifest or composition judged,
|
|
// said.
|
|
Candidates TryCandidates
|
|
// Inputs keeps the checking program's inputs (the state InputsStateName, served on InputVerb); BeginBound is how long a run ask may wait for a
|
|
// build node to begin it (DefaultBeginBound when zero).
|
|
Inputs Inputs
|
|
BeginBound time.Duration
|
|
Now func() time.Time
|
|
Logf func(format string, args ...any)
|
|
}
|
|
|
|
// TryCandidates answers the candidate the indexer holds for a try's group at its heads hash, waiting for it within the
|
|
// bound; an error says it did not come (and why).
|
|
type TryCandidates interface {
|
|
TryCandidate(ctx context.Context, group, heads string, bound time.Duration) (meshcatalog.CandidatesByGroupValue, error)
|
|
}
|
|
|
|
// CompositionReader composes a check: the engines' answers, which the rules judge (rules.JudgeEngines); an error is only
|
|
// a case that cannot be composed at all.
|
|
type CompositionReader interface {
|
|
Compose(ctx context.Context, c *Case) (rules.Composed, error)
|
|
}
|
|
|
|
func (j *Examiner) now() time.Time {
|
|
if j.Now != nil {
|
|
return j.Now()
|
|
}
|
|
return time.Now()
|
|
}
|
|
|
|
func (j *Examiner) logf(format string, args ...any) {
|
|
if j.Logf != nil {
|
|
j.Logf(format, args...)
|
|
}
|
|
}
|
|
|
|
// fetchBeside are the siblings as the fetch is asked to place them.
|
|
func fetchBeside(refs []BesideRef) []buildstep.FetchBeside {
|
|
out := make([]buildstep.FetchBeside, 0, len(refs))
|
|
for _, r := range refs {
|
|
out = append(out, buildstep.FetchBeside{Repository: r.Repository, Commit: r.Commit})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// besideLines are the siblings as the digest step echoes them: `<owner/repo> <commit>`, sorted.
|
|
func besideLines(refs []BesideRef) []string {
|
|
out := make([]string, 0, len(refs))
|
|
for _, r := range refs {
|
|
out = append(out, r.Repository+" "+r.Commit)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// target is a pull request on one build module: the node that pulled its ask, the workspace its fetch made and what it
|
|
// said.
|
|
type target struct {
|
|
pull Pull
|
|
module string
|
|
// language is the language of the toolchain the build module builds with (the `on` of its seat's acting rule).
|
|
language string
|
|
node string
|
|
toolchain string
|
|
// agreed is the toolchain the ask names, so a node off it declines the check (ADR 0584); nil when none is agreed.
|
|
agreed *buildstep.ToolchainUsed
|
|
fetched Fetched
|
|
// fetchArgs are the arguments of the fetch its check ask runs.
|
|
fetchArgs buildstep.Fetch
|
|
// priority are the Go packages (import paths) or the Node module directories whose test files the merge changes.
|
|
changed []string
|
|
// beside are the lines `<owner/repo> <commit>` of the siblings the workspace was given, sorted: its tests may read them,
|
|
// so a verdict is kept or taken only under a digest that says it folded exactly these in (lead decision L5).
|
|
beside []string
|
|
// runner is RunnerWorkspace when the target's ask runs the candidate's own build module (ADR 0535 §1), empty otherwise.
|
|
runner string
|
|
// judged is what the line says of the program that judged a change to the build module; empty for any other change.
|
|
judged string
|
|
}
|
|
|
|
// judgedBy takes what a step under runner workspace answered of the program that ran it (judged_by): the candidate, or the
|
|
// installed module (a module that does not take the runner says nothing, and is asked without it from now on: ADR 0535 §2).
|
|
// It says whether the candidate ran.
|
|
func (t *target) judgedBy(answer string) bool {
|
|
switch {
|
|
case strings.HasPrefix(answer, "candidate "):
|
|
t.judged = judgedByCandidate(t.module) + short12(strings.TrimPrefix(answer, "candidate "))
|
|
return true
|
|
case answer == "":
|
|
t.judged, t.runner = judgedByOldInstalled(t.module), ""
|
|
default:
|
|
t.judged, t.runner = judgedByInstalled(t.module), ""
|
|
}
|
|
return false
|
|
}
|
|
|
|
// stepArgs are a step's arguments with the target's runner.
|
|
func (t *target) stepArgs(args map[string]any) map[string]any {
|
|
if t.runner != "" {
|
|
args["runner"] = t.runner
|
|
}
|
|
return args
|
|
}
|
|
|
|
// state is one judging.
|
|
type state struct {
|
|
j *Examiner
|
|
c *Case
|
|
targets []*target
|
|
levels rules.Tally
|
|
// facts are the facts of the check met so far, in order: what the checking program judges (milestone 47/B0).
|
|
facts []rules.Step
|
|
// composition is how the check composed, recorded after every step (compose).
|
|
composition *rules.Composition
|
|
clone time.Duration
|
|
slow []string
|
|
// beside are the siblings read at their trunk's head, by repository (ADR 0548 §6).
|
|
beside map[string]string
|
|
// judgeHead is the group's mesh-delivery member head when its merge changes the judge (ADR 0578 §2); empty judges
|
|
// with mesh-delivery's trunk.
|
|
judgeHead string
|
|
// predefinedHead is the catalogue member's head when its merge changes predefined.json: the list is read there.
|
|
predefinedHead string
|
|
// unpreviewed are the members (pullKey) with no preview of a merge: a head that does not merge, or a fetch that did not
|
|
// answer. The indexer holds no manifest of them, so the assigner's composition would only wait: it is not awaited.
|
|
unpreviewed []string
|
|
// outOfStep are, by pull request (pullKey), the generated files a level's compare step found out of step (design 26).
|
|
outOfStep map[string][]string
|
|
// noCandidate says why the case has no previewed candidate of its heads from the indexer within the bound: for a whole
|
|
// try its manifests were not judged and it was not composed; for a pull request's check that began on the indexer's
|
|
// first write, its manifests were judged from that write and it was not composed. Empty when it has one, and for a pull
|
|
// request's check whose case already holds its fingerprints.
|
|
noCandidate string
|
|
}
|
|
|
|
// narrowTry says the case is a try narrowed to a level or to some tests: it judges no manifest, graph or composition.
|
|
func (st *state) narrowTry() bool { return st.c.Try != nil && !st.c.Try.Whole() }
|
|
|
|
// WaitsOnPreview says a candidate's module has no fingerprint yet because the indexer wrote it from a pull message alone,
|
|
// before it read the head's merge preview (meshcatalogclient.NoneNoPreview): its fingerprint comes with the preview, on the
|
|
// same heads, so a candidate that holds one is not yet the candidate of its heads previewed.
|
|
func WaitsOnPreview(m meshcatalog.CandidatesByGroupValuePullsItemFpsValue) bool {
|
|
return m.None2 == meshcatalogclient.NoneNoPreview
|
|
}
|
|
|
|
// previewPending says a module of the case's pulls waits on its merge preview (WaitsOnPreview).
|
|
func (st *state) previewPending() bool {
|
|
for _, p := range st.c.Pulls {
|
|
for _, m := range p.Fps {
|
|
if WaitsOnPreview(m) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// takeTryCandidate gives a whole try's pulls the fingerprints and manifests the indexer's candidate of its group holds,
|
|
// once the indexer wrote it from the try's previews. A pull request's check has them from its candidate (CaseOf), but the
|
|
// indexer writes a pull request's candidate first from its pull message alone and its fingerprints only with the merge
|
|
// preview, on the same heads: a check that began on that first write takes the previewed candidate too, before it judges
|
|
// the manifests, waits on the builds and composes (ADR 0603 §10; issue 1921: a check otherwise waited on no build and
|
|
// the engines, reading the previewed candidate, refused its modules as not yet built). A case with no such candidate is
|
|
// said so, in a note and in its composition, and is no pass (Tryer.run).
|
|
func (st *state) takeTryCandidate(ctx context.Context) {
|
|
if st.c.Try != nil && !st.c.Try.Whole() || st.c.Try == nil && !st.previewPending() {
|
|
return
|
|
}
|
|
no := func(why string) {
|
|
what := "the group's manifests and composition were not judged: "
|
|
if st.c.Try == nil {
|
|
what = "the composition was not judged (the manifests were judged from the indexer's first write): "
|
|
}
|
|
st.noCandidate = what + why
|
|
st.c.note(st.noCandidate)
|
|
}
|
|
if st.j.Candidates == nil {
|
|
no("the checker reads no candidate of the indexer's")
|
|
return
|
|
}
|
|
if len(st.unpreviewed) > 0 {
|
|
no("no preview of a merge of " + strings.Join(st.unpreviewed, ", ") + " (its head does not merge or its fetch did not answer)")
|
|
return
|
|
}
|
|
if st.c.Wait != nil {
|
|
st.c.Wait("the indexer's candidate of the group at its heads (up to " + st.j.buildWait().String() + ")")
|
|
defer st.c.Wait("")
|
|
}
|
|
cand, err := st.j.Candidates.TryCandidate(ctx, st.c.Group, st.c.HeadsHash, st.j.buildWait())
|
|
if err != nil {
|
|
no(err.Error())
|
|
return
|
|
}
|
|
for i, p := range st.c.Pulls {
|
|
for _, cp := range cand.Pulls {
|
|
if cp.Repository == p.Repository && cp.Head == p.Head {
|
|
st.c.Pulls[i].Modules, st.c.Pulls[i].Fps = cp.Modules, cp.Fps
|
|
}
|
|
}
|
|
}
|
|
st.c.FPSet = cand.Fpset
|
|
}
|
|
|
|
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
|
|
// be judged at all.
|
|
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
|
|
st := &state{j: j, c: c, levels: rules.NewTally(),
|
|
beside: map[string]string{}}
|
|
// Every member and level runs to its end, and every manifest the change touches is judged by the indexer's judge (ADR
|
|
// 0578 §2), also after a failure: the verdict names every failure, not only the first (the operator, 2026-11-02).
|
|
// Each failure is a fact recorded where it is met; the verdict is the judging rules' of the facts.
|
|
st.judgeAll(ctx)
|
|
st.takeTryCandidate(ctx)
|
|
st.seriesChanges(ctx) // DLV-C4: a fact of the check, named in the verdict whatever it is
|
|
// The heads' seat list, from the checking program, before the validator and the composer are asked (ADR 0636 §4).
|
|
// A list it cannot make ends the check there, failed naming the checking program, worded from the facts so far.
|
|
var answer rules.Answer
|
|
var by *meshdeliverychecker.VerdictsByGroupHeadsValueJudgedBy
|
|
if st.seatList(ctx) {
|
|
st.judgeManifests(ctx)
|
|
st.gatherGraph(ctx)
|
|
st.judgeRenames(ctx)
|
|
st.refuseUnparsed(ctx)
|
|
st.compose(ctx)
|
|
// The verdict is the checking program's of the facts (ADR 0644 §1, ADR 0656 §5).
|
|
answer, by = st.judged(ctx)
|
|
} else {
|
|
answer = rules.Answer{V: rules.AnswerVersion, Verdict: rules.Judge(st.factsOf())}
|
|
}
|
|
for _, n := range answer.Notes {
|
|
st.c.note(n)
|
|
}
|
|
answer.Verdict.JudgedBy = by
|
|
out := st.outcome(answer.Verdict)
|
|
out.Builds = st.j.buildNotes(ctx, st.c)
|
|
return out, nil
|
|
}
|
|
|
|
// compose records how the check composed as the facts' composition, after every step (ADR 0593 §7, §9): kept by a carry,
|
|
// not run (said why), or the engines' answers with the candidate's builds awaited first (ADR 0603 §10). A narrowed try
|
|
// and an examiner that composes nothing record none.
|
|
func (st *state) compose(ctx context.Context) {
|
|
if st.c.Composition != nil && st.c.Try == nil {
|
|
// A carry whose candidate's fingerprints did not change keeps the composition it passed (ADR 0548 §5).
|
|
kept := *st.c.Composition
|
|
st.composition = &rules.Composition{Kept: &kept}
|
|
st.c.note("the composition is kept: no fingerprint of a module it composed changed")
|
|
return
|
|
}
|
|
if st.narrowTry() {
|
|
return
|
|
}
|
|
if st.noCandidate != "" {
|
|
st.composition = &rules.Composition{NotRun: st.noCandidate}
|
|
return
|
|
}
|
|
if st.j.Compositions == nil {
|
|
return
|
|
}
|
|
if len(st.unpreviewed) > 0 {
|
|
// An engine would only say it cannot read a manifest the indexer will not have: the check does not ask it.
|
|
st.composition = &rules.Composition{NotRun: "no preview of a merge of " + strings.Join(st.unpreviewed, ", ") +
|
|
" (its head does not merge or its fetch did not answer), so the change cannot be composed"}
|
|
return
|
|
}
|
|
// ADR 0603 §10: the check composes with the candidate's builds; whether it failed already is the running rules' word
|
|
// on the facts so far, an asking choice only (the verdict is the rules' of every fact).
|
|
failed := rules.Judge(st.factsOf()).State == meshdeliverycheckerclient.VerdictFail
|
|
words, builds, late := st.awaitBuilds(ctx, failed)
|
|
c := &rules.Composition{Waits: words, BuildsFailed: builds, FPSet: st.c.FPSet}
|
|
st.composition = c
|
|
if late != "" {
|
|
c.NotRun = late // the builder's lateness, said by name, never the change's refusal (issue 1921)
|
|
return
|
|
}
|
|
comp, err := st.j.Compositions.Compose(ctx, st.c)
|
|
if err != nil {
|
|
// Said in the verdict, never left as a composition not yet run (review finding 4).
|
|
st.j.logf("%s: the composition could not be made, so it is said as not run: %v", st.c.Key, err)
|
|
c.NotRun = "not made: " + err.Error()
|
|
return
|
|
}
|
|
c.Composed = &comp
|
|
}
|
|
|
|
// outcome is the verdict judged from the facts with the slowest tests, the siblings read and what judged a build module.
|
|
func (st *state) outcome(v meshdeliverychecker.VerdictsByGroupHeadsValue) Outcome {
|
|
slow := rules.Slowest(st.levels)
|
|
out := Outcome{Verdict: v, Slowest: slow, OutOfStep: st.outOfStep}
|
|
if len(st.beside) > 0 {
|
|
out.Beside = st.beside
|
|
}
|
|
for _, t := range st.targets {
|
|
if t.judged != "" {
|
|
if out.Judged == nil {
|
|
out.Judged = map[string]string{}
|
|
}
|
|
out.Judged[pullKey(Pull{Repository: t.pull.Repository, Number: t.pull.Number})] = t.judged
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// stepOutOfStep keeps the out-of-step files a step of the pull request named, each once, sorted.
|
|
func (st *state) stepOutOfStep(p Pull, files []string) {
|
|
if len(files) == 0 {
|
|
return
|
|
}
|
|
if st.outOfStep == nil {
|
|
st.outOfStep = map[string][]string{}
|
|
}
|
|
k := pullKey(Pull{Repository: p.Repository, Number: p.Number})
|
|
all := append(st.outOfStep[k], files...)
|
|
slices.Sort(all)
|
|
st.outOfStep[k] = slices.Compact(all)
|
|
}
|
|
|
|
// fail records a failure the asking met as a fact of the check, where it is met, and answers it as a verdict of failure
|
|
// at a machine: the verdict is the judging rules' of the facts (rules.Judge), so a failure only returned is no failure
|
|
// (milestone 47/B0).
|
|
func (st *state) fail(machine, words string) {
|
|
st.facts = append(st.facts, rules.Step{Failed: &FailedFirst{Machine: machine, Words: words}})
|
|
}
|
|
|
|
// fail is a verdict of failure at a machine, for a step that answers one before any member is asked (tagged); it is
|
|
// recorded as facts (failedAll), never returned as the check's verdict.
|
|
func fail(machine, words string) *meshdeliverychecker.VerdictsByGroupHeadsValue {
|
|
v := failure(machine, words)
|
|
return &v
|
|
}
|
|
|
|
// failedAll records every failure a verdict of the asking names (its list, else its first) as facts of the check.
|
|
func (st *state) failedAll(v meshdeliverychecker.VerdictsByGroupHeadsValue) {
|
|
all := failedList(v.Failures)
|
|
if len(all) == 0 {
|
|
all = []FailedFirst{failedOf(v)}
|
|
}
|
|
for _, f := range all {
|
|
st.facts = append(st.facts, rules.Step{Failed: &f})
|
|
}
|
|
}
|
|
|
|
// factsOf are the check's facts as met: what the checking program judges.
|
|
func (st *state) factsOf() rules.Facts {
|
|
f := st.factsHead()
|
|
f.Steps, f.CloneMS, f.Composition, f.Notes = st.facts, st.clone.Milliseconds(), st.composition, st.c.Notes()
|
|
f.Series = st.c.allSeries()
|
|
return f
|
|
}
|
|
|
|
// levels are the levels a case's asks run: both, or the one a try names.
|
|
func (st *state) levelsOf() []string {
|
|
if st.c.Try != nil && st.c.Try.Level != "" {
|
|
return []string{st.c.Try.Level} // a try of one level runs that level alone
|
|
}
|
|
return []string{buildstep.LevelUnit, buildstep.LevelIntegration}
|
|
}
|
|
|
|
// judgeAll puts one check ask per pull request and build module the repository needs, the mesh-sdk member first, and
|
|
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails is a failure of that
|
|
// member; every other member is still judged, and the verdict names every failure (the operator, 2026-11-02), unjoined
|
|
// to its first lines (settled does that). Only a check whose context ended stops.
|
|
func (st *state) judgeAll(ctx context.Context) {
|
|
// The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members'
|
|
// Node fetches take the group's package (ADR 0500 §5).
|
|
pulls := slices.Clone(st.c.Pulls)
|
|
slices.SortStableFunc(pulls, func(a, b Pull) int {
|
|
if (a.Repository == SDKRepository) != (b.Repository == SDKRepository) {
|
|
if a.Repository == SDKRepository {
|
|
return -1
|
|
}
|
|
return 1
|
|
}
|
|
return 0
|
|
})
|
|
var groupPackage *buildstep.FetchPackage
|
|
met := len(st.facts)
|
|
for _, p := range pulls {
|
|
if ctx.Err() != nil {
|
|
if !st.failedSince(met) { // an ask that ended with the context said it already
|
|
st.fail(CheckerModule, ctx.Err().Error())
|
|
}
|
|
break
|
|
}
|
|
var merged []Fetched
|
|
modules := st.j.Modules.BuildModules(p.Repository)
|
|
for _, bm := range modules {
|
|
t := &target{pull: p, module: bm, language: st.j.languageOf(bm)}
|
|
if tc, none := buildstep.AgreedToolchain(st.j.Toolchains.Entries(t.language)); none == "" {
|
|
t.toolchain = tc.Version
|
|
t.agreed = &buildstep.ToolchainUsed{Language: t.language, Version: tc.Version, Recipe: tc.Recipe}
|
|
}
|
|
args := buildstep.Fetch{Repository: p.Repository, Commit: p.Head, Ref: refOf(p.Number), Base: p.Trunk}
|
|
if t.language == languageTypeScript {
|
|
if groupPackage != nil && p.Repository != SDKRepository {
|
|
args.Package = groupPackage
|
|
}
|
|
}
|
|
if t.language == languageGo {
|
|
beside, err := st.j.besideFor(ctx, st.c, p)
|
|
if err != nil {
|
|
st.fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
|
|
continue
|
|
}
|
|
if len(beside) > 0 {
|
|
args.Beside = fetchBeside(beside)
|
|
t.beside = besideLines(beside)
|
|
besideAtTrunk(st.c, beside, st.beside)
|
|
}
|
|
if p.Repository == catalogueRepository {
|
|
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
|
|
}
|
|
}
|
|
t.fetchArgs = args
|
|
st.targets = append(st.targets, t)
|
|
st.ask(ctx, t, &merged)
|
|
}
|
|
// Every build module's fetch merged (each is in merged once): the preview is written, also when the member's tests
|
|
// failed, so the indexer holds its manifests and the assigner can compose the change (review of 650a1d0).
|
|
switch {
|
|
case st.narrowTry(): // a try of a level or some tests owes the indexer no preview: it judges no manifest
|
|
case len(merged) == len(modules):
|
|
st.j.writePreview(ctx, p, merged)
|
|
default:
|
|
st.unpreviewed = append(st.unpreviewed, pullKey(p))
|
|
}
|
|
st.judgedByMember(p, merged)
|
|
// The group's package member packed by the Node fetch (ADR 0500 §5).
|
|
if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && st.j.changesPackage(p, merged) {
|
|
groupPackage = &buildstep.FetchPackage{Repository: p.Repository, Commit: p.Head, Ref: refOf(p.Number)}
|
|
}
|
|
}
|
|
}
|
|
|
|
// failedSince says a failure was recorded among the facts from the nth on.
|
|
func (st *state) failedSince(n int) bool {
|
|
for _, f := range st.facts[n:] {
|
|
if f.Failed != nil {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// DefaultDeclineBound is an Examiner's DeclineBound when it names none: longer than a check runs, so a node with the
|
|
// need that is busy is waited for, and short enough that a mesh where no node has it says so.
|
|
const DefaultDeclineBound = 30 * time.Minute
|
|
|
|
// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, then
|
|
// each level's digest, kept verdicts and tests. When the answer leaves packages unmet,
|
|
// their tests needing what that node lacks (issue 890), the passes are kept and the levels from the first such one are
|
|
// asked again of a node that has it (placed).
|
|
func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) {
|
|
st.askNeeding(ctx, t, merged, st.levelsOf(), nil, nil)
|
|
}
|
|
|
|
// askNeeding is ask with the levels to run, what the node must have and the packages it tests: none for the first ask
|
|
// of a target; for the placed one the unmet packages' needs and those packages alone, so no package the first ask judged
|
|
// is run or counted again (ADR 0552 §4).
|
|
func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, levels, needs, packages []string) {
|
|
p := t.pull
|
|
placed := len(needs) > 0
|
|
job := &buildstep.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages, Toolchain: t.agreed}
|
|
if st.j.Nodes != nil {
|
|
names, err := st.j.Nodes.Names(ctx)
|
|
if err == nil && len(names) == 0 {
|
|
err = fmt.Errorf("no node record is on the bus")
|
|
}
|
|
if err != nil {
|
|
st.fail(t.module, fmt.Sprintf("the check of %s#%d is not asked: the nodes' names the names check needs cannot be read: %v", p.Repository, p.Number, err))
|
|
return
|
|
}
|
|
job.NodeNames = names
|
|
}
|
|
if t.runner != "" {
|
|
job.Runner, job.RunnerWhen = t.runner, buildGoDir(st.goModule())
|
|
}
|
|
if st.c.Try != nil && st.c.Try.Run != "" {
|
|
job.Run, job.Kept = st.c.Try.Run, false // a try of some tests runs them alone: a partial run proves no package
|
|
}
|
|
kind := buildstep.AskCheck
|
|
if st.c.Kind == KindTrunk {
|
|
kind = buildstep.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
|
|
}
|
|
a := buildstep.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: st.j.askLanguageOf(t.module), Asker: CheckerModule,
|
|
At: st.j.now().UTC(), Check: job, Needs: needs}
|
|
if a.Language == buildstep.AskGo {
|
|
a.Packages = st.packagesOf(p.Repository)
|
|
}
|
|
var step buildstep.Step
|
|
var firstDecline time.Time
|
|
for {
|
|
what := fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number)
|
|
if placed {
|
|
what += " for a node with " + strings.Join(needs, ", ")
|
|
}
|
|
st.c.Progress.Set(what, "")
|
|
st.c.Asked(a.ID)
|
|
if err := st.j.Asks.Put(ctx, a); err != nil {
|
|
st.fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
|
|
return
|
|
}
|
|
waiting := ctx
|
|
if watch := st.c.Watch; watch != nil {
|
|
id := a.ID
|
|
watch(id, nil) // put: a try names it at once, not only when it ends (issue 1890)
|
|
waiting = buildrun.WithStepSeen(ctx, func(s buildstep.Step) { watch(id, &s) })
|
|
}
|
|
var err error
|
|
step, err = st.j.Asks.Wait(waiting, a)
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
st.fail(t.module, ctx.Err().Error())
|
|
return
|
|
}
|
|
st.fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
|
|
return
|
|
}
|
|
d, declined := buildstep.DeclinedOf(step)
|
|
if !declined {
|
|
break
|
|
}
|
|
// A node that lacks the need dropped the ask (issue 890): it is put again, for a node that has it to take; the
|
|
// node that declined looks at the stream again only after its next pull's wait.
|
|
now := st.j.now()
|
|
if firstDecline.IsZero() {
|
|
firstDecline = now
|
|
}
|
|
bound := st.j.DeclineBound
|
|
if bound <= 0 {
|
|
bound = DefaultDeclineBound
|
|
}
|
|
if now.Sub(firstDecline) > bound {
|
|
lacks := strings.Join(needs, ", ")
|
|
if lacks == "" {
|
|
lacks = strings.Join(d.Lacks, ", ") // a node off the agreed toolchain declined it (ADR 0584)
|
|
}
|
|
st.fail(step.Node, fmt.Sprintf("the check of %s#%d needs a node with %s, and none took it in %s: every node that pulled it lacks it (last %s)",
|
|
p.Repository, p.Number, lacks, bound, step.Node))
|
|
return
|
|
}
|
|
st.j.logf("%s#%d: the ask %s was declined on %s, which lacks %s; asked again", p.Repository, p.Number, a.ID, step.Node, strings.Join(d.Lacks, ", "))
|
|
a.ID, a.At = buildstep.NewID(), now.UTC()
|
|
}
|
|
// The fetch and the digests are what of the ask's time its levels' running did not take (Took.Clone).
|
|
took := time.Duration(0)
|
|
if !step.Started.IsZero() && step.Ended.After(step.Started) {
|
|
took = step.Ended.Sub(step.Started)
|
|
}
|
|
t.node = step.Node
|
|
st.facts = append(st.facts, rules.Step{Repository: p.Repository, Number: p.Number, Module: t.module, Node: step.Node})
|
|
st.c.Progress.Set(fmt.Sprintf("judge the check of %s#%d", p.Repository, p.Number), step.Node)
|
|
if step.State != buildstep.StepPassed {
|
|
st.fail(step.Node, fmt.Sprintf("the check of %s#%d %s: %s", p.Repository, p.Number, step.State, step.Reason))
|
|
return
|
|
}
|
|
var answer buildstep.CheckAnswer
|
|
if err := json.Unmarshal(step.Result, &answer); err != nil {
|
|
st.fail(step.Node, fmt.Sprintf("the check of %s#%d answered what is not a check: %s", p.Repository, p.Number, step.Result))
|
|
return
|
|
}
|
|
if t.fetched = answer.Fetch; t.fetched.Workspace == "" {
|
|
st.fail(step.Node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, step.Result))
|
|
return
|
|
}
|
|
if len(t.fetched.Conflicts) > 0 {
|
|
if !st.narrowTry() {
|
|
st.j.writePreview(ctx, p, append(*merged, t.fetched))
|
|
}
|
|
st.fail(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(t.fetched.Conflicts, ", ")))
|
|
return
|
|
}
|
|
t.changed = t.fetched.Changed
|
|
if !changesBuildGo(p.Repository, st.goModule(), t.changed) {
|
|
t.runner = "" // the build module used no runner: the merge does not change it
|
|
}
|
|
if !placed { // the placed ask fetched the same merge: its preview was read from the first
|
|
*merged = append(*merged, t.fetched)
|
|
st.checkBinariesOf(t) // the levels still run: the verdict names every failure
|
|
}
|
|
ran := 0.0
|
|
for _, la := range answer.Levels {
|
|
if placed {
|
|
if len(la.Unmet) > 0 {
|
|
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
|
st.fail(step.Node, fmt.Sprintf("the %s level of %s#%d left %s untested on %s, which was asked for a node with %s",
|
|
la.Level, p.Repository, p.Number, unmetNames(la.Unmet), step.Node, strings.Join(needs, ", ")))
|
|
return
|
|
}
|
|
la.Kept = nil // it names its packages and takes no other's kept verdict; this is only a guard against an older module
|
|
}
|
|
ran += st.level(ctx, t, la) // the next level was run too (buildstep): its failures are named as well
|
|
}
|
|
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
|
if unmet := buildstep.UnmetNeeds(answer.Levels); len(unmet) > 0 && !placed {
|
|
// Every level ran, its passes kept, and some packages were left to a node with what their tests need: the levels
|
|
// from the first that left one are asked of such a node, which takes the kept passes and tests those (issue 890).
|
|
var again, packages []string
|
|
seen := map[string]bool{}
|
|
for _, la := range answer.Levels {
|
|
if len(again) > 0 || len(la.Unmet) > 0 {
|
|
again = append(again, la.Level)
|
|
}
|
|
for _, u := range la.Unmet {
|
|
if !seen[u.Package] {
|
|
seen[u.Package] = true
|
|
packages = append(packages, u.Package)
|
|
}
|
|
}
|
|
}
|
|
st.j.logf("%s#%d: %s on %s left untested; asked of a node with %s", p.Repository, p.Number, unmetOf(answer.Levels), step.Node, strings.Join(unmet, ", "))
|
|
st.askNeeding(ctx, t, merged, again, unmet, packages)
|
|
}
|
|
}
|
|
|
|
// unmetNames names the unmet packages and their needs.
|
|
func unmetNames(unmet []buildstep.PackageNeeds) string {
|
|
var out []string
|
|
for _, u := range unmet {
|
|
out = append(out, u.Package+" (needs "+strings.Join(u.Needs, ", ")+")")
|
|
}
|
|
return strings.Join(out, ", ")
|
|
}
|
|
|
|
// unmetOf names the unmet packages of every level.
|
|
func unmetOf(levels []buildstep.LevelAnswer) string {
|
|
var out []string
|
|
for _, la := range levels {
|
|
if len(la.Unmet) > 0 {
|
|
out = append(out, "at "+la.Level+" "+unmetNames(la.Unmet))
|
|
}
|
|
}
|
|
return strings.Join(out, "; ")
|
|
}
|
|
|
|
// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
|
|
// of the packages that passed, the kept ones taken are counted, and every test result is read and counted, also after
|
|
// one that did not pass, so the verdict names every failed test of the level (issue 1921). It answers the level's
|
|
// running time (its tests' and setup's).
|
|
func (st *state) level(ctx context.Context, t *target, la buildstep.LevelAnswer) float64 {
|
|
if t.runner != "" && t.language == languageGo && la.Digest != nil {
|
|
t.judgedBy(la.Digest.JudgedBy) // the installed module judged it: the runner is not used again
|
|
}
|
|
if t.runner != "" && la.Judged != "" {
|
|
t.judgedBy(la.Judged) // the candidate judged the level: nothing is kept (ADR 0535 §4)
|
|
}
|
|
_, elapsed, results := st.levels.Level(t.pull.Repository, t.pull.Number, t.module, t.language, t.node, la)
|
|
st.facts = append(st.facts, rules.Step{Repository: t.pull.Repository, Number: t.pull.Number, Module: t.module, Language: t.language, Node: t.node, Level: rules.LevelFact(la)})
|
|
var digests map[string]string
|
|
if t.language == languageGo && la.Digest != nil {
|
|
// The packages come as a list, never keyed by a package path (issue 1930); an answer stored before reads too.
|
|
digests = la.Digest.Map()
|
|
}
|
|
for _, res := range results {
|
|
if t.runner != "" {
|
|
t.judgedBy(res.JudgedBy)
|
|
}
|
|
st.stepOutOfStep(t.pull, res.OutOfStep)
|
|
if len(digests) > 0 {
|
|
if _, err := st.j.Kept.KeepPasses(ctx, t.pull.Repository, la.Level, t.toolchain, res, digests, st.j.now()); err != nil {
|
|
st.j.logf("%s#%d: a passing package could not be kept: %v", t.pull.Repository, t.pull.Number, err)
|
|
}
|
|
}
|
|
}
|
|
return elapsed
|
|
}
|
|
|
|
// changesPackage says whether the merge of a pull request changes a file inside the named paths of one of its repository's
|
|
// TypeScript packages: a change of the manifest alone, or of files outside those paths, does not (ADR 0500 §5).
|
|
func (j *Examiner) changesPackage(p Pull, merged []Fetched) bool {
|
|
if j.Packages == nil {
|
|
return false
|
|
}
|
|
paths := j.Packages.PackagePaths(p.Repository)
|
|
for _, f := range merged {
|
|
for _, changed := range f.Changed {
|
|
for _, named := range paths {
|
|
named = strings.TrimSuffix(strings.TrimPrefix(named, "./"), "/")
|
|
if changed == named || strings.HasPrefix(changed, named+"/") {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|