Land feature 57's controller work on its release branch

The controller plans config, templates and moved providers' provisions,
hands bus accounts to the vault and judges the new manifest words, on the
feature's SDK and node-engine validator (feature 57, config and provisioning).
This commit is contained in:
2026-10-15 21:27:33 +00:00
parent fcdecb4361
commit 97a2d841c7
349 changed files with 42440 additions and 786 deletions
+51 -1
View File
@@ -173,6 +173,10 @@ type Reuse struct {
// keyFirst is a check's build (§1): its key is taken before anything is built, and a module with none is not
// built at all — nothing could ever take it.
keyFirst bool
// pull and head are the check's own pull request and the head it judged (novox/hq issue 703): it keeps only a
// build an earlier check of the same pull request at the same head made.
pull PullRef
head string
}
// errNoKey is a check's build of a module whose source does not pin it: not built (ADR 0350 §1).
@@ -414,7 +418,7 @@ func build(ctx context.Context, run Runner, publish Publisher,
// only when asked — a check, a dry run, `build --ref` and `rebuild` never are. Its key is taken by the same
// steps a build takes, stopped before anything is built; what it takes is copied by digest.
if reuse != nil && reuse.keyFirst {
_, key, err := keyFor(ctx, run, publish, manifest, within, workspace, commit, credentials, artifacts, args,
keyed, key, err := keyFor(ctx, run, publish, manifest, within, workspace, commit, credentials, artifacts, args,
held, npmrcPath, npmrc, seatBases, src, say)
if err == nil && key.Key == "" {
err = fmt.Errorf("%w: %s", errNoKey, key.Why)
@@ -423,6 +427,11 @@ func build(ctx context.Context, run Runner, publish Publisher,
say("reuse", "not built for the merge: %v", err)
return Result{Commit: commit, NoKey: err.Error()}, err
}
// **A check keeps a build of the same key a verdict names** (novox/hq issue 703): asked again when only
// the trunk moved, it builds nothing the earlier check built of an unchanged source.
if kept := keptCheckBuild(ctx, *reuse, manifest.Module, key.Key, len(artifacts), say); kept != nil {
built, src, reused = kept.Made, keyed, kept
}
} else if reuse != nil && reuse.Scratch != nil {
taken, keyed, why := takeACheckBuild(ctx, run, publish, *reuse, manifest, within, workspace, commit,
credentials, artifacts, args, held, npmrcPath, npmrc, seatBases, src, onTrunk, say)
@@ -553,6 +562,47 @@ func takeACheckBuild(ctx context.Context, run Runner, publish Publisher, reuse R
return taken, keyed, ""
}
// keptCheckBuild is the build a check keeps instead of building (novox/hq issue 703): the one kept under the
// module's key by an earlier check of the same pull request at the same head — only the trunk moved — when a verdict
// of the build seat names its key and every digest — the record alone is never trusted (ADR 0350 §2) — and it holds
// every artifact the module declares. Another pull request's build, or an earlier head's, is never kept. Nil, and
// the check builds, otherwise.
func keptCheckBuild(ctx context.Context, reuse Reuse, module, key string, artifacts int,
say func(step, format string, args ...any)) *Reused {
if reuse.Scratch == nil || reuse.head == "" {
return nil
}
var same []NamedBuild
for _, n := range reuse.Named {
if n.Pull == reuse.pull {
same = append(same, n)
}
}
if len(same) == 0 {
return nil
}
found, ok, err := reuse.Scratch.Find(ctx, module, key)
switch {
case err != nil:
say("reuse", "the scratch namespace could not be read, so the check builds: %v", err)
return nil
case !ok:
return nil
case found.Pull != reuse.pull || found.Head == "" || twelve(found.Head) != twelve(reuse.head):
say("reuse", "the build kept under %s is %s, not this pull request's at this head: built", twelve(key[3:]),
found.Said())
return nil
case unnamed(same, module, key, found) != "":
say("reuse", "a build is kept under %s and no verdict names it with its digests: built", twelve(key[3:]))
return nil
case len(found.Made) != artifacts:
return nil
}
kept := &Reused{From: found, Made: found.Made}
say("reuse", "kept, not built: %s", strings.Replace(kept.Words(), "reused from", "the build of", 1))
return kept
}
// keyFor takes a build's reuse key by the steps a build takes, each artifact stopped before anything is built or
// published (ADR 0350 §2): its own copy of the source inputs, so the build's are untouched.
func keyFor(ctx context.Context, run Runner, publish Publisher, manifest catalogue.Manifest,
+616 -40
View File
@@ -13,7 +13,9 @@ import (
"os/exec"
"path/filepath"
"regexp"
"sort"
"strings"
"sync"
"time"
"github.com/novox/mesh-controller/internal/artifacts"
@@ -87,6 +89,19 @@ type CheckSpec struct {
// Judge is who judges the gate: "" the controller the mesh runs, "self" the change's own, "validator"
// the running one with the change's validator.
Judge string
// Release is the release branch the pull request merges into (novox/hq ADR 0358), empty for any other base. The
// gate is then judged by the controller beside it as mesh-controller — mesh-controller's release branch, never
// the controller the mesh runs nor its main — with mesh-host's validator from beside it when mesh-host was read at
// the release branch too; a change to the controller still judges itself and a change to the node-engine brings
// its own validator. The verdict names the branch, the judge's commit and the branch each beside was read at.
Release string
// JudgeCache is the judge cache (novox/hq ADR 0358 §6): a directory of the build agent's own, outside the
// workspace, that only judge builds mount — never a check's containers, so no pull request's script can write it.
// It holds the Go build cache every judge is built with (judges stay warm), the release-branch judges kept by the
// commits they are built from (judges/, one per release-branch head), and each judge build's fresh home (homes/,
// removed after the build). Empty: each judge is built cold under the check's own root, which every check empties
// before it clones, and nothing is kept.
JudgeCache string
// Toolchain is the image a check's Go runs in: the mesh's own Go toolchain, as it holds it.
// Toolchains is every toolchain the mesh holds, by language, for a script that declares another.
Toolchain string
@@ -153,10 +168,12 @@ func ToolchainsOf(held map[string]string) map[string]string {
return out
}
// Beside is one repository cloned next to the one checked.
// Beside is one repository cloned next to the one checked: at Ref, or at Else when Else is said and the repository
// has no branch Ref (novox/hq ADR 0358 §3).
type Beside struct {
Repository string
Ref string
Else string
}
// CheckVerdict is what came of one check. Verdict and Summary are the gate's; Gate and Repo each layer.
@@ -179,6 +196,16 @@ type CheckVerdict struct {
Members []JudgedHead
// Built are the builds it made for its merge (novox/hq ADR 0350 §1), each with its key, or why not.
Built []CheckBuilt
// Release is the release branch it was judged for (novox/hq ADR 0358 §4), JudgedBy the commit of the controller
// that judged its gate (twelve characters), and BesideAt the branch each repository beside it was read at, by
// directory. Empty for a check of any other base.
Release string
JudgedBy string
BesideAt map[string]string
// Validator says which node-engine validator the judge of a release-branch check was built with (ADR 0358 §2):
// mesh-host's of the release branch with its commit, the one the release branch's controller vendors, the
// change's own, or the one the change's own controller vendors. Empty for a check of any other base.
Validator string
}
// Layer is one layer of a check, judged.
@@ -274,7 +301,17 @@ func ScriptToolchain(script []byte) string {
// Check runs one merge check. An error is that it could not run; the verdict is then "error".
func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry string, forge GitCredential,
log Log) (CheckVerdict, error) {
// Everything a check says goes to the build's log, which the bus keeps: said redacted (novox/hq issue 462).
// Everything a check says goes to the build's log, which the bus keeps: said redacted (novox/hq issue 462). A line
// at a time: its builds for the merge say theirs while its tests run (novox/hq issue 716).
if log != nil {
var saying sync.Mutex
inner := log
log = func(step, message string) {
saying.Lock()
defer saying.Unlock()
inner(step, message)
}
}
redact := redactorFor(forge)
say := logging(func(step, message string) {
if log != nil {
@@ -303,13 +340,32 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
if err := forgetLeftCredentials(workspace); err != nil {
return CheckVerdict{}, err
}
clone := func(repository, ref, dir string) error {
// **mesh-lab's replays run where no pull request reaches** (novox/hq issue 709, ADR 0358 §6): they hold the
// container runtime's socket, so their sources, home, Go cache and module cache are none of the workspace's.
replay, err := replayDirs(spec.JudgeCache, workspace, root, spec.ID)
if err != nil {
return CheckVerdict{}, err
}
if replay.base != "" {
defer os.RemoveAll(replay.base)
}
// cloneAt clones a repository and checks out ref — or, when the repository has no branch ref and orElse is said,
// orElse (novox/hq ADR 0358 §3); readAt keeps which, by directory.
readAt := map[string]string{}
cloneAt := func(repository, ref, orElse, dir string) error {
if _, err := run(ctx, root, "git", cloneWith(credentials, "clone", "--quiet", repository, dir)...); err != nil {
return fmt.Errorf("cannot clone %s: %s", redact.redact(repository), redact.redact(err.Error()))
}
if err := recordedWithoutUserinfo(ctx, run, filepath.Join(root, dir), repository); err != nil {
return err
}
if orElse != "" && ref != "" {
if _, err := run(ctx, filepath.Join(root, dir), "git", "rev-parse", "--verify", "--quiet",
"refs/remotes/origin/"+ref); err != nil {
ref = orElse
}
}
readAt[dir] = ref
if ref != "" {
if _, err := run(ctx, filepath.Join(root, dir), "git", "checkout", "--quiet", ref); err != nil {
return fmt.Errorf("%s has no %s: %w", redact.redact(repository), ref, err)
@@ -317,6 +373,9 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
}
return nil
}
clone := func(repository, ref, dir string) error {
return cloneAt(repository, ref, "", dir)
}
name := spec.Repo
if name == "" || !safeName.MatchString(name) {
name = "checked"
@@ -352,8 +411,28 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
}
named := func(v CheckVerdict) CheckVerdict {
v.Head, v.Trunk, v.Merge, v.Beside, v.Members = judged.Head, judged.Trunk, judged.Merge, judged.Beside, judged.Members
v.Release, v.JudgedBy, v.BesideAt, v.Validator = spec.Release, judged.JudgedBy, judged.BesideAt, judged.Validator
return v
}
// **What the merge would move is built from a checkout nothing of the check touches** (novox/hq issue 703): the
// gate and the repository's own check run in containers that mount the workspace read-write, so they can change
// its tree, its objects and its caches. The builds are made from a copy of the merge's objects, cloned before
// anything of the change runs, outside the workspace — so what is kept under a key is what the key describes. Each
// build's working copy is cloned from it into the workspace only after the check has ended (forTheMerge): the
// host's container engine sees no other directory of the holder's (novox/hq issue 713).
buildingTree := ""
if len(spec.Builds) > 0 && spec.BuildWith != nil && len(spec.Group) == 0 {
at := judged.Merge
if at == "" {
at = spec.Ref
}
var building string
building, buildingTree, err = checkoutForTheBuilds(ctx, run, tree, at, workspace)
if err != nil {
return CheckVerdict{}, err
}
defer os.RemoveAll(building)
}
// **The base branch's check judges the change** (novox/hq issue 310), never the one the change brings.
judging, err := TheCheckThatJudges(ctx, tree, spec.Base)
if err != nil {
@@ -376,9 +455,35 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
if dir == name || !safeName.MatchString(dir) {
continue
}
if err := clone(b.Repository, b.Ref, dir); err != nil {
if dir == "mesh-lab" {
// **mesh-lab is never read at a release branch** (novox/hq ADR 0358): its replays run with the container
// runtime's socket, so a branch of a release's name — which anyone may create — is never what runs there,
// whether it is asked with a fallback or without.
if b.Else != "" || (spec.Release != "" && b.Ref == spec.Release) {
b.Ref = b.Else
if b.Ref == "" {
b.Ref = "main"
}
}
b.Else = ""
}
if err := cloneAt(b.Repository, b.Ref, b.Else, dir); err != nil {
return CheckVerdict{}, fmt.Errorf("beside it, %w", err)
}
if dir == "mesh-lab" {
// The copy the replays run from, made before anything of the change runs, outside every check container
// (novox/hq issue 709) — and whether it holds replays, read now, so nothing written later decides it.
if err := replay.copyLab(ctx, run, filepath.Join(root, dir)); err != nil {
return CheckVerdict{}, fmt.Errorf("mesh-lab could not be copied for its replays: %w", err)
}
}
if spec.Release != "" {
// The branch each was read at is named with its commit (ADR 0358 §4).
if judged.BesideAt == nil {
judged.BesideAt = map[string]string{}
}
judged.BesideAt[dir] = readAt[dir]
}
// The commit each one read is named in the verdict (ADR 0323 §3): its move is issue 568's question.
if at := headAt(ctx, run, filepath.Join(root, dir)); at != "" {
if judged.Beside == nil {
@@ -386,7 +491,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
}
judged.Beside[dir] = at
}
say("check", "beside it %s at %s", dir, short(b.Ref))
say("check", "beside it %s at %s", dir, refWords(readAt[dir]))
}
// A delivery group's other heads (novox/hq ADR 0239), each at its head, for the gate to compose with this.
@@ -438,6 +543,25 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
// Every clone is made: the credential is gone before anything of the check runs (novox/hq issue 462).
forget()
// **What the merge moves is built beside the tests** (novox/hq issue 716) when the build agent has a judge cache to
// build it in, where no container of the check reaches; without one, after the check has judged.
logged := func(step, message string) {
if log != nil {
log(step, redact.redact(message))
}
}
var beside *besideTheTests
if buildingTree != "" {
place, err := buildsBesideTheTests(spec.JudgeCache, workspace, spec.ID)
if err != nil {
return CheckVerdict{}, err
}
if place != "" {
beside = startBesideTheTests(ctx, run, spec, place, buildingTree, judged, forge, logged, say)
defer beside.abandon()
}
}
// The facts, and the versions they say the mesh runs.
if registry == "" {
return CheckVerdict{}, errors.New("no artifact store to read the facts snapshot from")
@@ -500,21 +624,56 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
user = fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid())
}
in := func(image, dir string, env []string, command ...string) []string {
home := workspace
for _, e := range env {
if strings.HasPrefix(e, "HOME=") {
home = strings.TrimPrefix(e, "HOME=")
}
}
// As the builder itself: what a check writes into the workspace is the builder's to remove.
args := []string{"run", "--rm", "--network", "host", "--volume", workspace + ":" + workspace, "--workdir", dir,
"--user", user, "--env", "HOME=" + workspace,
"--user", user, "--env", "HOME=" + home,
// The check's own checkouts, whoever cloned them: git in a container of another user than the
// one that cloned refuses a repository it does not own ("dubious ownership"), and Go's build
// stamps the version from git — a judge that would not build for want of it.
"--env", "GIT_CONFIG_COUNT=1", "--env", "GIT_CONFIG_KEY_0=safe.directory", "--env", "GIT_CONFIG_VALUE_0=*"}
for _, e := range env {
args = append(args, "--env", e)
if !strings.HasPrefix(e, "HOME=") {
args = append(args, "--env", e)
}
}
return append(append(args, image), command...)
}
inToolchain := func(dir string, env []string, command ...string) []string {
return in(spec.Toolchain, dir, env, command...)
}
// **Every judge is built where no pull request can reach** (novox/hq issue 482, ADR 0358 review): the workspace's
// Go cache, HOME and git configuration are written by pull requests' own scripts, and git runs during a Go build
// to stamp its version — a planted hook would rewrite the judge's sources before they compile. So a judge is built
// with a HOME and configuration made fresh for this check and a Go cache only judge builds mount (judgeBuildDirs),
// no global or system git configuration, no Go environment file, and no version stamping from git.
judgeHome, judgeGoCache, err := judgeBuildDirs(spec.JudgeCache, workspace, root, spec.ID)
if err != nil {
return CheckVerdict{}, err
}
defer os.RemoveAll(judgeHome)
inJudgeBuild := func(dir string, _ []string, command ...string) []string {
args := in(spec.Toolchain, dir, JudgeBuildEnv(judgeHome, judgeGoCache), command...)
if spec.JudgeCache == "" {
return args // under the check's root, in the workspace mounted already
}
// The judge cache's parts this build needs, mounted into judge builds alone (ADR 0358 §6).
return append([]string{args[0], "--volume", judgeHome + ":" + judgeHome, "--volume",
judgeGoCache + ":" + judgeGoCache}, args[1:]...)
}
inReplays := func(env []string, command ...string) []string {
args := in(spec.Toolchain, filepath.Join(replay.lab, "replays"), append(ReplayEnv(replay.home, replay.goCache,
replay.modCache), env...), command...)
// The socket goes to the replays alone, never to the change's own code.
mounts := []string{"--volume", "/var/run/docker.sock:/var/run/docker.sock", "--volume", replay.base + ":" +
replay.base, "--volume", replay.goCache + ":" + replay.goCache, "--volume", replay.modCache + ":" + replay.modCache}
return append(append([]string{args[0]}, mounts...), args[1:]...)
}
var out tail
// running runs one container of the check, its output into the report, in a process group of its own.
running := func(args []string) error {
@@ -526,10 +685,23 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
// The judge. Its failing to build is the change's fault when the change is the judge or its validator,
// and the check's when it is the controller the mesh runs.
gate, judgeFault, err := judgeFor(ctx, labelled, run, spec, root, tree, inToolchain, say)
gate, judgeFault, err := judgeFor(ctx, labelled, run, spec, root, tree, judged.BesideAt, inJudgeBuild, say)
if err != nil {
return CheckVerdict{}, err
}
if spec.Release != "" {
// **A release-branch judgement says so** (novox/hq ADR 0358 §4): the branch, the controller's commit that
// judged, and each repository read beside it at its branch and commit.
judgedBy := filepath.Join(root, "mesh-controller")
if spec.Judge == "self" {
judgedBy = tree
}
judged.JudgedBy = headAt(ctx, run, judgedBy)
judged.Validator = releaseValidator(spec, judged.BesideAt, judged.Beside)
said := releaseWords(spec.Release, judged.JudgedBy, judged.Validator, judged.BesideAt, judged.Beside)
fmt.Fprintf(&out, "--- %s\n", said)
say("check", "%s", said)
}
verdictFile := filepath.Join(root, "verdict.json")
env := []string{EnvFacts + "=" + factsFile, EnvGate + "=" + gate, EnvGateStore + "=" + storeURL,
EnvTestStore + "=" + storeURL, EnvTestBus + "=nats://" + bus, EnvRepository + "=" + spec.Owner + "/" + spec.Repo,
@@ -555,7 +727,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
say("check", "the gate: %d module(s) of the graph touched — %s", len(modules), strings.Join(modules, ", "))
fmt.Fprintf(&out, "--- the gate: %s\n", strings.Join(modules, ", "))
v.Gate.Verdict, v.Gate.Summary = gateLayer(ctx, spec, tree, root, gate, verdictFile, env, inToolchain,
running, &out, bus, workspace, name, say)
running, &out, bus, name, replay.hasReplays, replay.refused(), inReplays, say)
}
if timedOut() {
v.Gate.Verdict, v.Gate.Summary = "error", fmt.Sprintf("the check ran past %s and was ended", CheckTimeout)
@@ -595,6 +767,11 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
}
}
if spec.Release != "" {
// The statuses name the release branch and the judge first, so the forge's short description keeps them.
mark := releaseMark(spec.Release, judged.JudgedBy)
v.Gate.Summary, v.Repo.Summary = mark+v.Gate.Summary, mark+v.Repo.Summary
}
// What the check printed travels in the verdict to the forge and the controller: redacted as the log is.
v.Gate.Summary, v.Repo.Summary, v.Repo.Failed = redact.redact(v.Gate.Summary), redact.redact(v.Repo.Summary),
redact.redact(v.Repo.Failed)
@@ -605,24 +782,68 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
// **What the merge would move, built once, here** (novox/hq ADR 0350 §1): after the judging, from the merge it
// judged, into the scratch namespace — so the walk of the merge commit takes it rather than building again.
if len(spec.Builds) > 0 && spec.BuildWith != nil && len(spec.Group) == 0 {
if why := skipsTheBuilds(v); why != "" {
say("check", "%s", why)
} else {
began := time.Now()
v.Built = buildForTheMerge(ctx, run, spec, *spec.BuildWith, workspace, tree, judged.Merge, judged, forge,
func(step, message string) {
if log != nil {
log(step, redact.redact(message))
}
}, say)
say("check", "built for the merge: %d of %d module(s) (%s)", builtCount(v.Built), len(v.Built),
time.Since(began).Round(time.Second))
}
switch {
case beside != nil:
v.Built = beside.settle(ctx, v)
case len(spec.Builds) > 0 && spec.BuildWith != nil && len(spec.Group) == 0:
v.Built = forTheMerge(ctx, run, spec, v, workspace, buildingTree, judged, forge, logged, say)
}
return named(v), nil
}
// forTheMerge builds what the merge moves once the check has judged it, from merged — the copy of the merge
// checkoutForTheBuilds made outside the workspace — **in the builder's own workspace** (novox/hq issue 713). A build's
// containers are run by the host's container engine, which mounts a path as the host names it: the build seat's
// holder runs in a container that shares only its workspace with the host, at the same path, so a working copy in
// its own temporary directory reaches the toolchain as an empty directory ("go.mod file not found"), every check
// build failed, and every walk built again. The copy stays outside the workspace, so nothing the tests ran reaches it;
// the working copy cloned from it is made only now, after every container of the check has ended.
func forTheMerge(ctx context.Context, run Runner, spec CheckSpec, verdict CheckVerdict, workspace, merged string,
judged CheckVerdict, forge GitCredential, log Log, say func(step, format string, args ...any)) []CheckBuilt {
if why := skipsTheBuilds(verdict); why != "" {
say("check", "%s", why)
return nil
}
began := time.Now()
built := buildForTheMerge(ctx, run, spec, *spec.BuildWith, workspace, merged, judged.Merge, judged, forge, log, say)
say("check", "built for the merge: %d of %d module(s) (%s)", builtCount(built), len(built),
time.Since(began).Round(time.Second))
return built
}
// checkoutForTheBuilds is a clone of the commit a check builds for its merge, made before anything of the check runs,
// outside the workspace its containers mount (novox/hq issue 703): the directory to remove, and the clone in it.
// Cloned over git's own transport, never by linking the objects: what the check's containers later write into the
// workspace's objects cannot reach it.
func checkoutForTheBuilds(ctx context.Context, run Runner, tree, at, workspace string) (string, string, error) {
dir, err := os.MkdirTemp("", "mesh-check-builds-")
if err != nil {
return "", "", err
}
if withinDir(workspace, dir) {
os.RemoveAll(dir)
return "", "", fmt.Errorf("the temporary directory %s is inside the workspace %s, which a check's containers "+
"mount: the merge is not built from where its tests could change it", dir, workspace)
}
commit, err := run(ctx, tree, "git", "rev-parse", "--verify", at+"^{commit}")
if err != nil {
os.RemoveAll(dir)
return "", "", fmt.Errorf("the commit to build for the merge, %s: %w", at, err)
}
commit = strings.TrimSpace(commit)
if _, err := run(ctx, tree, "git", "update-ref", "refs/heads/mesh-check-builds", commit); err != nil {
os.RemoveAll(dir)
return "", "", err
}
cloned := filepath.Join(dir, "merged")
if _, err := run(ctx, dir, "git", "clone", "--quiet", "--no-local", "--no-checkout", "--branch",
"mesh-check-builds", tree, cloned); err != nil {
os.RemoveAll(dir)
return "", "", fmt.Errorf("cannot copy the merge to build it: %w", err)
}
return dir, cloned, nil
}
// builtCount is how many of a check's builds for its merge were built.
func builtCount(built []CheckBuilt) int {
n := 0
@@ -794,8 +1015,9 @@ const SomeManifestsEnv = "MESH_MODULE_CHECK_SOME=1"
// gateLayer runs the gate: the touched manifests through `module check`, every machine composed with the
// change, and the replays of what the mesh runs. It answers the gate's verdict and summary.
func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFile string, env []string,
inToolchain func(string, []string, ...string) []string, running func([]string) error, out *tail, bus, workspace,
name string, say func(step, format string, args ...any)) (string, string) {
inToolchain func(string, []string, ...string) []string, running func([]string) error, out *tail, bus,
name string, hasReplays bool, replaysRefused string, inReplays func([]string, ...string) []string,
say func(step, format string, args ...any)) (string, string) {
// 1. The manifests the change touches, as the judge reads them: a manifest it cannot read, or one with a
// problem the change brings, fails here. **What was already so on the base branch is said and fails
// nothing** — the gate's own rule: a module whose manifest the running controller already finds fault
@@ -887,24 +1109,37 @@ func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFil
// 3. **The replays of what the mesh runs** (to-be 45 §9, M9): mesh-lab's, from its main — reviewed code,
// so given the container runtime the resolver replay raises containers with — against the bus of the
// release the mesh runs and the change's own catalogue when the change is to the catalogue.
if lab := filepath.Join(root, "mesh-lab", "replays"); ctx.Err() == nil {
if _, err := os.Stat(lab); err == nil {
if ctx.Err() == nil && hasReplays && replaysRefused != "" {
// **Never from the workspace** (novox/hq issue 709): without the judge cache there is no place outside it.
say("check", "%s", replaysRefused)
fmt.Fprintf(out, "--- the replays: %s\n", replaysRefused)
return "error", replaysRefused
}
if ctx.Err() == nil {
if hasReplays {
catalogue := filepath.Join(root, "mesh-catalog")
if name == "mesh-catalog" {
catalogue = tree
}
say("check", "the replays of what the mesh runs, from mesh-lab")
fmt.Fprintln(out, "--- the replays (mesh-lab replays/)")
args := inToolchain(lab, []string{EnvTestBus + "=nats://" + bus, "MESH_REPLAY_CATALOGUE=" + catalogue,
"GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")},
// In the replays' own environment (inReplays): reviewed code, the socket, and nothing the workspace holds
// but the catalogue they read.
args := inReplays([]string{EnvTestBus + "=nats://" + bus, "MESH_REPLAY_CATALOGUE=" + catalogue},
"go", "test", "-count=1", "./...")
// The socket goes to the replays alone, never to the change's own code above.
args = append([]string{args[0], "--volume", "/var/run/docker.sock:/var/run/docker.sock"}, args[1:]...)
if err := running(args); err != nil {
if ctx.Err() != nil {
return "error", "the check was ended during the replays"
}
return "fail", "a replay of a core incident fails with this change: " + lastLine(out.String())
if replaysNotStarted(err) {
// The container runtime could not start them (a mount, an image): the check's fault, never the change's.
said := "the replays could not be started, so the change is not judged by them: " + lastLine(out.String())
say("check", "%s", said)
return "error", said
}
said := "a replay of a core incident fails with this change: " + lastLine(out.String())
say("check", "%s", said)
return "fail", said
}
}
}
@@ -919,21 +1154,32 @@ func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFil
// the mesh runs. It answers the judge's path; or, when the change makes its own judge unbuildable, why —
// the change's fault, a failing gate; or an error when the check cannot build a judge at all. A check
// whose gate does not run builds a judge only to hand its scripts one, and goes on without when it cannot.
func judgeFor(ctx context.Context, labelled, run Runner, spec CheckSpec, root, tree string,
//
// **A pull request into a release branch is judged by that branch's controller** (novox/hq ADR 0358 §2): the one
// beside it as mesh-controller, read at the release branch — with mesh-host's validator from beside it when mesh-host
// has that branch too, or with the change's own validator for a change to the node-engine — never the controller the
// mesh runs. besideAt is the branch each repository beside it was read at.
func judgeFor(ctx context.Context, labelled, run Runner, spec CheckSpec, root, tree string, besideAt map[string]string,
inToolchain func(string, []string, ...string) []string, say func(step, format string, args ...any)) (string, string, error) {
gated := spec.Gated()
switch spec.Judge {
case "self":
bin := filepath.Join(root, "bin", "judge-of-itself")
if _, err := labelled(ctx, root, "docker", inToolchain(tree,
[]string{"CGO_ENABLED=0", "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOCACHE=" + filepath.Join(filepath.Dir(root), "go-cache")},
if _, err := labelled(ctx, root, "docker", inToolchain(tree, nil,
"go", "build", "-o", bin, "./cmd/mesh-controller")...); err != nil {
return "", "the change's controller, which judges itself, does not build: " + firstLine(err.Error()), nil
}
say("check", "judged by the change's own controller")
return bin, "", nil
}
bin, judgeTree, err := judge(ctx, labelled, run, root, inToolchain, say)
var bin, judgeTree string
var err error
if spec.Release != "" {
withHost := spec.Judge != "validator" && besideAt["mesh-host"] == spec.Release
bin, judgeTree, err = releaseJudge(ctx, labelled, run, spec, root, withHost, inToolchain, say)
} else {
bin, judgeTree, err = judge(ctx, labelled, run, root, inToolchain, say)
}
if err != nil {
if !gated {
say("check", "no judge for its script: %v", err)
@@ -952,15 +1198,346 @@ func judgeFor(ctx context.Context, labelled, run Runner, spec CheckSpec, root, t
}
}
withValidator := filepath.Join(root, "bin", "judge-with-this-validator")
if _, err := labelled(ctx, root, "docker", inToolchain(filepath.Join(root, judgeTree),
[]string{"CGO_ENABLED=0", "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOCACHE=" + filepath.Join(filepath.Dir(root), "go-cache")},
if _, err := labelled(ctx, root, "docker", inToolchain(filepath.Join(root, judgeTree), nil,
"go", "build", "-o", withValidator, "./cmd/mesh-controller")...); err != nil {
if spec.Release != "" {
return "", spec.Release + "'s controller does not build with this change's validator: " +
firstLine(err.Error()), nil
}
return "", "the controller the mesh runs does not build with this change's validator: " + firstLine(err.Error()), nil
}
if spec.Release != "" {
say("check", "judged by %s's controller, with this change's validator", spec.Release)
return withValidator, "", nil
}
say("check", "judged by the controller the mesh runs, with this change's validator")
return withValidator, "", nil
}
// releaseJudge builds the controller that judges a pull request into a release branch (novox/hq ADR 0358 §2): the
// one beside the check as mesh-controller, read at that branch — with mesh-host's validator from beside it in place
// of the one it vendors when withHost. **Built once per release-branch head**: kept in the spec's judge cache by the
// commits it is built from and taken from there while they stand. It answers the binary and the directory it was
// built from.
func releaseJudge(ctx context.Context, run, plain Runner, spec CheckSpec, root string, withHost bool,
inToolchain func(string, []string, ...string) []string, say func(step, format string, args ...any)) (string, string, error) {
const dir = "mesh-controller"
if _, err := os.Stat(filepath.Join(root, dir)); err != nil {
return "", "", fmt.Errorf("no controller of %s beside the check to judge it with", spec.Release)
}
key := fullHeadAt(ctx, plain, filepath.Join(root, dir))
if key == "" {
return "", "", fmt.Errorf("the controller of %s beside the check names no commit", spec.Release)
}
with := ""
if withHost {
host := fullHeadAt(ctx, plain, filepath.Join(root, "mesh-host"))
if host == "" {
return "", "", fmt.Errorf("mesh-host of %s beside the check names no commit", spec.Release)
}
key += "-with-mesh-host-" + host
with = ", with mesh-host's validator of " + spec.Release + " at " + twelve(host)
}
bin := filepath.Join(root, "bin", "mesh-controller")
if took := judgeKept(keptJudges(spec.JudgeCache), key, bin); took {
say("check", "judged by %s's controller at %s%s, built for an earlier check of that head", spec.Release,
twelve(key), with)
return bin, dir, nil
}
if withHost {
vendored := filepath.Join(root, dir, "vendor", "github.com", "novox", "mesh-host")
for _, pkg := range []string{"validate", filepath.Join("internal", "declaration")} {
if err := replaceGoFiles(filepath.Join(root, "mesh-host", pkg), filepath.Join(vendored, pkg)); err != nil {
return "", "", fmt.Errorf("mesh-host's validator of %s could not be put in its controller: %w", spec.Release, err)
}
}
}
// Built where no pull request reaches (JudgeBuildEnv): a judge kept for this head is the head's, nothing else.
if _, err := run(ctx, root, "docker", inToolchain(filepath.Join(root, dir), nil,
"go", "build", "-o", bin, "./cmd/mesh-controller")...); err != nil {
return "", "", fmt.Errorf("the controller of %s does not build%s: %w", spec.Release, with, err)
}
// Kept before anything of the change runs: nothing a pull request's script writes can reach the cache.
keepJudge(keptJudges(spec.JudgeCache), key, bin, say)
say("check", "judged by %s's controller at %s%s", spec.Release, twelve(key), with)
return bin, dir, nil
}
// JudgeBuildEnv is the environment every judge is built in (novox/hq ADR 0358 §6, issue 482): home, made fresh for the
// build, as HOME and configuration; goCache, the judge cache's Go cache that no check's container mounts; no global
// or system git configuration; no Go environment file; and no version stamped from git, so no git runs at all.
func JudgeBuildEnv(home, goCache string) []string {
return []string{"HOME=" + home, "XDG_CONFIG_HOME=" + filepath.Join(home, ".config"),
"GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_NOSYSTEM=1", "GOENV=off",
"GOCACHE=" + goCache, "GOPATH=" + filepath.Join(home, "go"),
"CGO_ENABLED=0", "GOFLAGS=-mod=vendor -buildvcs=false", "GOPROXY=off"}
}
// judgeBuildDirs are the home and the Go cache a check's judge builds use (ADR 0358 §6). With a judge cache: a home
// made fresh under its homes/ for this check, and its go-cache/, kept warm between checks — both on the host path the
// build agent declares, so the container runtime mounts what the agent sees, and nothing is left in a temporary
// directory of the host. Without one: both under the check's root, which every check empties before it clones. A
// judge cache inside the workspace is refused: a check's containers mount the workspace.
func judgeBuildDirs(cache, workspace, root, id string) (string, string, error) {
if cache == "" {
home, goCache := filepath.Join(root, "judge-home"), filepath.Join(root, "judge-go-cache")
for _, d := range []string{home, goCache} {
if err := os.MkdirAll(d, 0o700); err != nil {
return "", "", err
}
}
return home, goCache, nil
}
if withinDir(workspace, cache) {
return "", "", fmt.Errorf("the judge cache %s is inside the workspace %s, which a check's containers mount: "+
"no judge is built where a pull request could reach it", cache, workspace)
}
if !safeName.MatchString(id) {
return "", "", fmt.Errorf("the check's id %q cannot name its place in the judge cache", id)
}
home, goCache := filepath.Join(cache, "homes", id), filepath.Join(cache, "go-cache")
if err := os.RemoveAll(home); err != nil {
return "", "", err
}
for _, d := range []string{home, goCache} {
if err := os.MkdirAll(d, 0o700); err != nil {
return "", "", err
}
}
return home, goCache, nil
}
// replayPlace is where mesh-lab's replays run from and with (novox/hq issue 709), never anything a pull request can
// write: base, the directory made for this check under the judge cache's homes/ (the same path on the host and in the
// build agent), removed after it; lab, the copy of mesh-lab they run; home, their HOME and configuration; and the judge
// cache's Go build and module caches. Empty without a judge cache: the replays are then refused. hasReplays says whether
// mesh-lab held replays when it was cloned, before anything of the change ran.
type replayPlace struct {
base, lab, home, goCache, modCache string
hasReplays bool
}
// replaysNeedTheJudgeCache is the gate's error when mesh-lab holds replays and the build agent has no judge cache.
const replaysNeedTheJudgeCache = "the replays need the build agent's judge cache (mesh-catalog#297); refused rather " +
"than run from the workspace"
// refused is why the replays cannot run here, empty when they can.
func (p replayPlace) refused() string {
if p.base == "" {
return replaysNeedTheJudgeCache
}
return ""
}
// replayDirs makes the replays' place (issue 709, ADR 0358 §6). The replays' build is as trusted as a judge's: it
// compiles mesh-lab's trunk, reviewed code, and nothing of the pull request, which hands them only its catalogue, as
// data. Never the workspace: the gate runs before the replays, a pull request's own code among its containers. Without
// a judge cache there is no place: the build agent declares no other host path, and the replays are refused.
func replayDirs(cache, workspace, root, id string) (replayPlace, error) {
if cache == "" {
return replayPlace{}, nil
}
if withinDir(workspace, cache) {
return replayPlace{}, fmt.Errorf("the judge cache %s is inside the workspace %s, which a check's containers "+
"mount: no replay is run where a pull request could reach it", cache, workspace)
}
if !safeName.MatchString(id) {
return replayPlace{}, fmt.Errorf("the check's id %q cannot name its place in the judge cache", id)
}
base := filepath.Join(cache, "homes", id+"-replays")
p := replayPlace{base: base, lab: filepath.Join(base, "mesh-lab"), home: filepath.Join(base, "home"),
goCache: filepath.Join(cache, "go-cache"), modCache: filepath.Join(cache, "go-modules")}
if err := os.RemoveAll(base); err != nil {
return replayPlace{}, err
}
for _, d := range []string{p.home, p.goCache, p.modCache} {
if err := os.MkdirAll(d, 0o700); err != nil {
return replayPlace{}, err
}
}
return p, nil
}
// copyLab reads whether mesh-lab's checkout, freshly cloned and before anything of the change runs, holds replays, and
// copies it over git's own transport to where the replays run it — when there is such a place.
func (p *replayPlace) copyLab(ctx context.Context, run Runner, from string) error {
_, err := os.Stat(filepath.Join(from, "replays"))
p.hasReplays = err == nil
if p.lab == "" {
return nil
}
return copyCheckout(ctx, run, from, p.lab)
}
// replaysNotStarted says the container runtime could not start the replays' container — its own exit status 125, or
// 126 and 127 for a command it could not run — rather than that a replay failed.
func replaysNotStarted(err error) bool {
var exit *exec.ExitError
if !errors.As(err, &exit) {
return true
}
code := exit.ExitCode()
return code == 125 || code == 126 || code == 127
}
// ReplayEnv is the environment mesh-lab's replays run in (novox/hq issue 709): home as HOME and configuration, the Go
// build and module caches given, no global or system git configuration, no Go environment file, and no version
// stamped from git — as a judge is built (JudgeBuildEnv), with the modules the replays fetch.
func ReplayEnv(home, goCache, modCache string) []string {
return []string{"HOME=" + home, "XDG_CONFIG_HOME=" + filepath.Join(home, ".config"),
"GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_NOSYSTEM=1", "GOENV=off",
"GOCACHE=" + goCache, "GOMODCACHE=" + modCache, "GOPATH=" + filepath.Join(home, "go"),
"GOFLAGS=-buildvcs=false", "GOWORK=off"}
}
// copyCheckout copies a checkout at the commit it stands at into dir, over git's own transport — never by linking
// objects — so what is written into the source afterwards does not reach the copy.
func copyCheckout(ctx context.Context, run Runner, from, dir string) error {
commit := fullHeadAt(ctx, run, from)
if commit == "" {
return fmt.Errorf("%s names no commit", from)
}
if _, err := run(ctx, filepath.Dir(dir), "git", "clone", "--quiet", "--no-local", "--no-checkout", from, dir); err != nil {
return err
}
_, err := run(ctx, dir, "git", "checkout", "--quiet", "--detach", commit)
return err
}
// keptJudges is where a judge cache keeps the release-branch judges, empty when there is no judge cache.
func keptJudges(cache string) string {
if cache == "" {
return ""
}
return filepath.Join(cache, "judges")
}
// judgesKept is how many release-branch judges a judge cache keeps: a head moved past is not built from again.
const judgesKept = 4
// judgeKept puts the judge kept under key in the cache at bin, and says whether there was one.
func judgeKept(cache, key, bin string) bool {
if cache == "" {
return false
}
body, err := os.ReadFile(filepath.Join(cache, key, "mesh-controller"))
if err != nil {
return false
}
if err := os.MkdirAll(filepath.Dir(bin), 0o755); err != nil {
return false
}
if err := os.WriteFile(bin, body, 0o755); err != nil {
return false
}
now := time.Now()
_ = os.Chtimes(filepath.Join(cache, key), now, now)
return true
}
// keepJudge keeps a judge just built under key, and lets go of all but the judgesKept most recently used. A judge
// that cannot be kept is said and built again next time: the cache only saves time.
func keepJudge(cache, key, bin string, say func(step, format string, args ...any)) {
if cache == "" {
return
}
body, err := os.ReadFile(bin)
if err == nil {
err = os.MkdirAll(filepath.Join(cache, key), 0o755)
}
if err == nil {
kept := filepath.Join(cache, key, "mesh-controller")
if err = os.WriteFile(kept+".part", body, 0o755); err == nil {
err = os.Rename(kept+".part", kept)
}
}
if err != nil {
say("check", "the judge could not be kept for the next check, which builds it again: %v", err)
return
}
entries, err := os.ReadDir(cache)
if err != nil || len(entries) <= judgesKept {
return
}
type used struct {
name string
at time.Time
}
var all []used
for _, e := range entries {
if info, err := e.Info(); err == nil && e.IsDir() {
all = append(all, used{e.Name(), info.ModTime()})
}
}
sort.Slice(all, func(i, j int) bool { return all[i].at.After(all[j].at) })
for _, u := range all[min(judgesKept, len(all)):] {
if u.name != key {
_ = os.RemoveAll(filepath.Join(cache, u.name))
}
}
}
// fullHeadAt is the whole commit a checkout stands at, empty when it names none.
func fullHeadAt(ctx context.Context, run Runner, tree string) string {
out, err := run(ctx, tree, "git", "rev-parse", "--verify", "--quiet", "HEAD^{commit}")
if err != nil {
return ""
}
return strings.TrimSpace(out)
}
// releaseValidator says which node-engine validator a release-branch check's judge is built with (novox/hq ADR 0358
// §2), by the rule judgeFor builds it by.
func releaseValidator(spec CheckSpec, at, commits map[string]string) string {
switch {
case spec.Judge == "self":
return "the validator the change's own controller vendors"
case spec.Judge == "validator":
return "the change's own validator"
case at["mesh-host"] == spec.Release:
return fmt.Sprintf("mesh-host's validator of %s at %s", spec.Release, commits["mesh-host"])
}
return fmt.Sprintf("the validator %s's controller vendors (mesh-host has no branch %s)", spec.Release, spec.Release)
}
// releaseWords is how a release-branch check says what judged it and what it read (novox/hq ADR 0358 §4).
func releaseWords(release, judgedBy, validator string, at, commits map[string]string) string {
var read []string
for _, dir := range sortedKeys(at) {
read = append(read, fmt.Sprintf("%s %s at %s", dir, refWords(at[dir]), commits[dir]))
}
said := fmt.Sprintf("judged for the release branch %s by its controller at %s, with %s", release, judgedBy, validator)
if len(read) > 0 {
said += "; read beside it: " + strings.Join(read, ", ")
}
return said
}
// releaseMark opens each layer's summary of a release-branch check, so a status says it first.
func releaseMark(release, judgedBy string) string {
return fmt.Sprintf("[release %s, judged by its controller %s] ", release, judgedBy)
}
// refWords is a ref as a line says it: a commit by its first eight characters, a branch whole.
func refWords(ref string) string {
if commitLike.MatchString(ref) {
return short(ref)
}
return ref
}
// commitLike is a ref that names a commit.
var commitLike = regexp.MustCompile(`^[0-9a-f]{7,40}$`)
func sortedKeys(m map[string]string) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
// replaceGoFiles puts a package's Go files — never its tests — in place of another copy's.
func replaceGoFiles(from, into string) error {
old, err := filepath.Glob(filepath.Join(into, "*.go"))
@@ -1195,8 +1772,7 @@ func judge(ctx context.Context, run, plain Runner, root string, inToolchain func
say func(step, format string, args ...any)) (string, string, error) {
bin := filepath.Join(root, "bin", "mesh-controller")
build := func(dir string) error {
_, err := run(ctx, root, "docker", inToolchain(filepath.Join(root, dir),
[]string{"CGO_ENABLED=0", "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOCACHE=" + filepath.Join(filepath.Dir(root), "go-cache")},
_, err := run(ctx, root, "docker", inToolchain(filepath.Join(root, dir), nil,
"go", "build", "-o", bin, "./cmd/mesh-controller")...)
return err
}
+269 -65
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"os"
"path/filepath"
"sync"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
@@ -35,8 +36,17 @@ type CheckBuilding struct {
Publish Publisher
Scratch Scratch
Held map[string]string
// Running is each artifact of the build the mesh has registered for its module, keyed "<module>/<artifact>"
// (novox/hq issue 721): what a build for the merge is compared with — never Held's newest build, which may be one
// its gate failed and the mesh put back.
Running map[string]string
Npmrc Npmrc
Seats map[string]string
// Named are the builds the build seat's verdicts of earlier checks of this pull request at this head name
// (novox/hq issue 703): a build kept under a module's key is kept again, instead of built, only when one of them
// names its key and every digest, and the build is this pull request's at this head. The record alone is never
// enough.
Named []NamedBuild
}
// CheckBuilt is one module a check built for its merge, as its verdict lists it (§1): its key and its artifacts, or
@@ -48,6 +58,14 @@ type CheckBuilt struct {
// NotBuilt says why it was not: no key, a build that failed, or a module after one of those.
NotBuilt string
Took time.Duration
// Kept is the check whose build this check kept instead of building, its key unchanged (novox/hq issue 703):
// the verdict names that build, by key and digest, as that check's. Empty when this check built it.
Kept string
// Same says every artifact came out the same as the build the mesh holds of the module (novox/hq issue 721), and
// SameAs names, per artifact, the held reference it was compared with: the walk leaves the module where it is
// while the mesh still holds that build. Empty when it differs, or could not be compared.
Same string
SameAs map[string]string
}
// skipsTheBuilds is why a check builds nothing for its merge; empty when it builds.
@@ -60,13 +78,45 @@ func skipsTheBuilds(v CheckVerdict) string {
return ""
}
// buildForTheMerge builds each module the plan moves, tier by tier, into the scratch namespace, and keeps what it
// built under its keys. A module standing after one not built is not built either: what it stands on in the walk
// is not what it would stand on here. tree is the check's clone and merge the merge commit it made there.
// buildForTheMerge builds each module the plan moves into the scratch namespace (makeForTheMerge), and keeps what it
// built under its keys (keepForTheMerge): the order of a check that builds after it has judged.
func buildForTheMerge(ctx context.Context, run Runner, spec CheckSpec, with CheckBuilding, workspace, tree, merge string,
judged CheckVerdict, forge GitCredential, log Log, say func(step, format string, args ...any)) []CheckBuilt {
out, kept := makeForTheMerge(ctx, run, spec, with, workspace, tree, merge, judged, forge, log, say)
return keepForTheMerge(ctx, spec, with, out, kept, say)
}
// makeForTheMerge builds each module the plan moves, tier by tier, into the scratch namespace — **the modules of a
// tier at the same time** (novox/hq issue 716): none of them stands on another, and a tier waits only for the ones
// before it, whose builds it is handed. A module standing after one not built is not built either: what it stands on
// in the walk is not what it would stand on here. tree is the check's copy of the merge and merge the merge commit it
// made there; workspace is where each build's working copy is made, one directory per module, which the host's
// container engine sees at the same path (novox/hq issue 713). Answers what each module came to, in the plan's order,
// and the builds to keep — nothing is kept here.
func makeForTheMerge(ctx context.Context, run Runner, spec CheckSpec, with CheckBuilding, workspace, tree, merge string,
judged CheckVerdict, forge GitCredential, log Log, say func(step, format string, args ...any)) ([]CheckBuilt, []CheckBuild) {
if len(spec.Builds) == 0 || with.Publish == nil || with.Scratch == nil {
return nil
return nil, nil
}
// What an earlier check left, ended before it could remove it, is not built on.
place := filepath.Join(workspace, "for-the-merge")
_ = os.RemoveAll(place)
defer os.RemoveAll(place)
// One log, said to by builds running at once: a line at a time.
var saying sync.Mutex
if log != nil {
inner := log
log = func(step, message string) {
saying.Lock()
defer saying.Unlock()
inner(step, message)
}
}
inner := say
say = func(step, format string, args ...any) {
saying.Lock()
defer saying.Unlock()
inner(step, format, args...)
}
ref := merge
if ref == "" {
@@ -76,76 +126,230 @@ func buildForTheMerge(ctx context.Context, run Runner, spec CheckSpec, with Chec
for k, v := range with.Held {
held[k] = v
}
var out []CheckBuilt
out := make([]CheckBuilt, len(spec.Builds))
results := make([]Result, len(spec.Builds))
var kept []CheckBuild
stopAt, stopWhy := -1, ""
for _, b := range spec.Builds {
began := time.Now()
if stopAt >= 0 && b.Tier > stopAt {
out = append(out, CheckBuilt{Module: b.Module, NotBuilt: stopWhy})
continue
for start := 0; start < len(spec.Builds); {
tier := spec.Builds[start].Tier
end := start
for end < len(spec.Builds) && spec.Builds[end].Tier == tier {
end++
}
if b.Skip != "" {
say("build", "%s not built for the merge: %s", b.Module, b.Skip)
out = append(out, CheckBuilt{Module: b.Module, NotBuilt: b.Skip})
if stopAt < 0 || b.Tier < stopAt {
stopAt, stopWhy = b.Tier, "it stands after "+b.Module+", which was not built for the merge"
// What this tier's builds are handed: the builds of the tiers before it, read alone while they run.
handed := map[string]string{}
for k, v := range held {
handed[k] = v
}
var building sync.WaitGroup
for i := start; i < end; i++ {
b := spec.Builds[i]
if stopAt >= 0 && b.Tier > stopAt {
out[i] = CheckBuilt{Module: b.Module, NotBuilt: stopWhy}
continue
}
continue
}
repository, at := b.Repository, b.Ref
if b.Merged {
repository, at = tree, ref
}
say("build", "for the merge: %s (tier %d)", b.Module, b.Tier)
var mirrored []string
result, err := build(ctx, run, scratchPublisher{to: with.Publish}, repository, b.Path, at,
filepath.Join(workspace, "for-the-merge"), held, with.Npmrc, forge, log, &mirrored, &Reuse{keyFirst: true},
with.Seats)
built := CheckBuilt{Module: b.Module, Key: result.ReuseKey, Took: time.Since(began)}
switch {
case errors.Is(err, errNoKey):
built.NotBuilt = err.Error()
case err != nil:
built.NotBuilt = "its build failed: " + err.Error()
case result.ReuseKey == "":
built.NotBuilt = "it has no reuse key: " + result.NoKey
}
if built.NotBuilt != "" {
say("build", "%s not built for the merge: %s", b.Module, built.NotBuilt)
out = append(out, built)
if stopAt < 0 || b.Tier < stopAt {
stopAt, stopWhy = b.Tier, "it stands after "+b.Module+", which was not built for the merge"
if b.Skip != "" {
say("build", "%s not built for the merge: %s", b.Module, b.Skip)
out[i] = CheckBuilt{Module: b.Module, NotBuilt: b.Skip}
continue
}
continue
}
built.Made = result.Built
for _, m := range result.Built {
held[b.Module+"/"+m.Name] = m.Reference
}
say("build", "%s built for the merge under %s (%s)", b.Module, twelve(result.ReuseKey[3:]), built.Took.Round(time.Second))
out = append(out, built)
if len(result.Built) > 0 {
kept = append(kept, CheckBuild{Key: result.ReuseKey, Module: b.Module, Parts: result.ReuseParts,
Made: result.Built, Pull: PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number},
Head: judged.Head, Trunk: judged.Trunk, Check: spec.ID, At: time.Now().UTC()})
}
}
_ = os.RemoveAll(filepath.Join(workspace, "for-the-merge"))
if len(kept) > 0 {
if err := with.Scratch.Keep(ctx, PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number}, spec.ID,
time.Now().UTC(), kept); err != nil {
// Built and not findable: the walk builds, and says no check built it. Said, never the check's verdict.
say("build", "the builds for the merge could not be kept, and the walk will build: %v", err)
if err := with.Scratch.Discard(ctx, kept); err != nil {
say("build", "and what they made could not all be let go of: %v", err)
repository, at := b.Repository, b.Ref
if b.Merged {
repository, at = tree, ref
}
for i := range out {
if out[i].NotBuilt == "" {
out[i].NotBuilt = fmt.Sprintf("built, and could not be kept: %v", err)
name := b.Module
if !safeName.MatchString(name) {
name = fmt.Sprintf("module-%d", i)
}
building.Add(1)
go func(i int, b CheckBuildSpec, repository, at, dir string) {
defer building.Done()
began := time.Now()
say("build", "for the merge: %s (tier %d)", b.Module, b.Tier)
var mirrored []string
result, err := build(ctx, run, scratchPublisher{to: with.Publish}, repository, b.Path, at, dir, handed,
with.Npmrc, forge, log, &mirrored, &Reuse{keyFirst: true, Scratch: with.Scratch, Named: with.Named,
pull: PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number}, head: judged.Head},
with.Seats)
built := CheckBuilt{Module: b.Module, Key: result.ReuseKey, Took: time.Since(began)}
switch {
case errors.Is(err, errNoKey):
built.NotBuilt = err.Error()
case err != nil:
built.NotBuilt = "its build failed: " + err.Error()
case result.ReuseKey == "":
built.NotBuilt = "it has no reuse key: " + result.NoKey
}
if built.NotBuilt == "" {
// **Compared with what runs** (novox/hq issue 721): a failed comparison says nothing, and the
// module moves as before.
same, against, err := sameAsHeld(ctx, run, dir, b.Module, result.Built, with.Running)
switch {
case err != nil:
say("build", "%s could not be compared with the build the mesh holds, so it moves: %v", b.Module, err)
case same != "":
built.Same, built.SameAs = same, against
say("build", "%s %s", b.Module, same)
}
}
out[i], results[i] = built, result
}(i, b, repository, at, filepath.Join(place, name))
}
building.Wait()
for i := start; i < end; i++ {
b, built, result := spec.Builds[i], &out[i], results[i]
if built.NotBuilt != "" {
if built.Key != "" || built.Took > 0 {
say("build", "%s not built for the merge: %s", b.Module, built.NotBuilt)
}
if stopAt < 0 || b.Tier < stopAt {
stopAt, stopWhy = b.Tier, "it stands after "+b.Module+", which was not built for the merge"
}
continue
}
built.Made = result.Built
for _, m := range result.Built {
held[b.Module+"/"+m.Name] = m.Reference
}
if result.Reused != nil {
// **Kept, not built** (novox/hq issue 703): the earlier check's build, named as that check's, so the
// walk takes it as the verdict names it; this check's document names it too, so it stays while this
// check does (ADR 0350 §5).
built.Kept = result.Reused.From.Check
say("build", "%s kept for the merge under %s: %s", b.Module, twelve(result.ReuseKey[3:]),
result.Reused.From.Said())
kept = append(kept, result.Reused.From)
continue
}
say("build", "%s built for the merge under %s (%s)", b.Module, twelve(result.ReuseKey[3:]),
built.Took.Round(time.Second))
if len(result.Built) > 0 {
kept = append(kept, CheckBuild{Key: result.ReuseKey, Module: b.Module, Parts: result.ReuseParts,
Made: result.Built, Pull: PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number},
Head: judged.Head, Trunk: judged.Trunk, Check: spec.ID, At: time.Now().UTC()})
}
}
start = end
}
return out, kept
}
// keepForTheMerge keeps a check's builds under their keys, its own document naming them, so the walk of its merge
// finds them; what cannot be kept is let go of, said, and each module says it — never the check's verdict.
func keepForTheMerge(ctx context.Context, spec CheckSpec, with CheckBuilding, out []CheckBuilt, kept []CheckBuild,
say func(step, format string, args ...any)) []CheckBuilt {
if len(kept) == 0 {
return out
}
if err := with.Scratch.Keep(ctx, PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number}, spec.ID,
time.Now().UTC(), kept); err != nil {
// Built and not findable: the walk builds, and says no check built it. Said, never the check's verdict.
say("build", "the builds for the merge could not be kept, and the walk will build: %v", err)
if err := with.Scratch.Discard(ctx, kept); err != nil {
say("build", "and what they made could not all be let go of: %v", err)
}
for i := range out {
if out[i].NotBuilt == "" {
out[i].NotBuilt = fmt.Sprintf("built, and could not be kept: %v", err)
}
}
}
return out
}
// discardForTheMerge lets go of builds a check made beside its tests when the check did not pass (novox/hq issue 716):
// a change that fails is not merged, and nothing it built is kept. A build another check names stays (Discard).
func discardForTheMerge(ctx context.Context, with CheckBuilding, kept []CheckBuild, why string,
say func(step, format string, args ...any)) {
say("check", "%s: the %d build(s) made beside it are let go of", why, len(kept))
if len(kept) == 0 {
return
}
if err := with.Scratch.Discard(ctx, kept); err != nil {
say("build", "what they made could not all be let go of, and goes with the week's sweep: %v", err)
}
}
// besideTheTests are a check's builds for its merge running while its gate and its own check run (novox/hq issue
// 716): started once every clone is made, waited for when the check has judged, kept when it passed and let go of
// when it did not.
type besideTheTests struct {
spec CheckSpec
with CheckBuilding
cancel context.CancelFunc
done chan struct{}
place string
out []CheckBuilt
kept []CheckBuild
began time.Time
settled bool
say func(step, format string, args ...any)
}
// buildsBesideTheTests is where a check's builds run beside its tests: a directory made for this check under the judge
// cache's homes/ (ADR 0358 §6), on the host path the build agent declares — so the host's container engine mounts what
// the agent sees (issue 713) — and never mounted by a check's container, so nothing a pull request's script runs can
// change a build while it is made (issue 703). Empty, and the builds wait for the check, without a judge cache.
func buildsBesideTheTests(cache, workspace, id string) (string, error) {
if cache == "" {
return "", nil
}
if withinDir(workspace, cache) {
return "", fmt.Errorf("the judge cache %s is inside the workspace %s, which a check's containers mount: "+
"nothing is built for the merge where a pull request could reach it", cache, workspace)
}
if !safeName.MatchString(id) {
return "", fmt.Errorf("the check's id %q cannot name its place in the judge cache", id)
}
place := filepath.Join(cache, "homes", id+"-builds")
if err := os.RemoveAll(place); err != nil {
return "", err
}
if err := os.MkdirAll(place, 0o700); err != nil {
return "", err
}
return place, nil
}
// startBesideTheTests starts the builds for the merge in place, from merged, the check's copy of the merge.
func startBesideTheTests(ctx context.Context, run Runner, spec CheckSpec, place, merged string, judged CheckVerdict,
forge GitCredential, log Log, say func(step, format string, args ...any)) *besideTheTests {
building, cancel := context.WithCancel(ctx)
b := &besideTheTests{spec: spec, with: *spec.BuildWith, cancel: cancel, done: make(chan struct{}), place: place,
began: time.Now(), say: say}
say("check", "building what the merge moves (%d module(s)) beside its tests", len(spec.Builds))
go func() {
defer close(b.done)
b.out, b.kept = makeForTheMerge(building, run, spec, b.with, place, merged, judged.Merge, judged, forge, log, say)
}()
return b
}
// settle waits for the builds and keeps them when the check passed; a check that did not pass lets go of them, and
// its verdict names none.
func (b *besideTheTests) settle(ctx context.Context, verdict CheckVerdict) []CheckBuilt {
<-b.done
b.settled = true
defer os.RemoveAll(b.place)
if why := skipsTheBuilds(verdict); why != "" {
discardForTheMerge(ctx, b.with, b.kept, why, b.say)
return nil
}
built := keepForTheMerge(ctx, b.spec, b.with, b.out, b.kept, b.say)
b.say("check", "built for the merge: %d of %d module(s), beside the tests (%s)", builtCount(built), len(built),
time.Since(b.began).Round(time.Second))
return built
}
// abandon ends builds a check that ended without a verdict leaves, and lets go of what they made.
func (b *besideTheTests) abandon() {
if b.settled {
return
}
b.cancel()
<-b.done
defer os.RemoveAll(b.place)
letGo, done := context.WithTimeout(context.Background(), time.Minute)
defer done()
discardForTheMerge(letGo, b.with, b.kept, "the check ended without a verdict", b.say)
}
+50
View File
@@ -210,3 +210,53 @@ func TestACheckMergeRunsNoHook(t *testing.T) {
t.Fatalf("a hook ran when the check made its merge: %s", said)
}
}
// What a check's tests change in the workspace never reaches what it builds for the merge (novox/hq issue 703): the
// builds are made from a copy of the merge, outside the workspace, cloned before anything of the change runs.
func TestWhatTheTestsChangeInTheWorkspaceNeverReachesTheBuild(t *testing.T) {
repo, head, _ := aTrunkAndABranch(t, map[string]string{"main.go": "package main // the merge\n"},
map[string]string{"trunk-only": "t"}, map[string]string{"head-only": "h"})
workspace := t.TempDir()
tree := filepath.Join(workspace, "check", "checked")
if out, err := exec.Command("git", "clone", "--quiet", repo, tree).CombinedOutput(); err != nil {
t.Fatalf("%v\n%s", err, out)
}
dir, cloned, err := checkoutForTheBuilds(t.Context(), Command, tree, head, workspace)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
if withinDir(workspace, cloned) {
t.Fatalf("the builds' checkout %s is inside the workspace %s the tests mount", cloned, workspace)
}
// The tests rewrite the tree, its branch and every object it holds.
if err := os.WriteFile(filepath.Join(tree, "main.go"), []byte("package main // planted\n"), 0o644); err != nil {
t.Fatal(err)
}
if err := filepath.WalkDir(filepath.Join(tree, ".git", "objects"), func(path string, d os.DirEntry, err error) error {
if err != nil || d.IsDir() {
return err
}
_ = os.Chmod(path, 0o644)
return os.WriteFile(path, []byte("planted"), 0o644)
}); err != nil {
t.Fatal(err)
}
// The build clones the merge from its own checkout, as buildForTheMerge does, and reads what the merge holds.
build := filepath.Join(t.TempDir(), "source")
if out, err := exec.Command("git", "clone", "--quiet", cloned, build).CombinedOutput(); err != nil {
t.Fatalf("%v\n%s", err, out)
}
if out, err := exec.Command("git", "-C", build, "checkout", "--quiet", head).CombinedOutput(); err != nil {
t.Fatalf("%v\n%s", err, out)
}
body, err := os.ReadFile(filepath.Join(build, "main.go"))
if err != nil {
t.Fatal(err)
}
if string(body) != "package main // the merge\n" {
t.Fatalf("the build read %q: what the tests wrote reached it", body)
}
}
+870
View File
@@ -0,0 +1,870 @@
package builder
import (
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
// **A pull request into a release branch is judged by the controller of that branch** (novox/hq ADR 0358, issue
// 704): the build seat's check builds the judge from mesh-controller read at the release branch — with mesh-host's
// validator of that branch when mesh-host has one — reads every repository beside it at that branch where it has one
// and at its trunk otherwise, and names the branch, the judge's commit and each repository read with its commit.
const releaseBranch = "feat/57-config-and-provisioning"
// gitInRepo runs git in a repository made for a test.
func gitInRepo(t *testing.T, dir string, args ...string) string {
t.Helper()
cmd := exec.Command("git", args...)
cmd.Dir = dir
cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org",
"GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org")
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("git %v: %v\n%s", args, err, out)
}
return strings.TrimSpace(string(out))
}
// commitFiles writes files into a repository and commits them on the branch checked out.
func commitFiles(t *testing.T, dir string, files map[string]string) string {
t.Helper()
for name, body := range files {
if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil {
t.Fatal(err)
}
}
gitInRepo(t, dir, "add", "-A")
gitInRepo(t, dir, "commit", "--quiet", "-m", "x")
return gitInRepo(t, dir, "rev-parse", "HEAD")
}
// aRepositoryWithARelease is a repository whose main holds mainFiles and, when releaseFiles is not nil, a release
// branch over them holding releaseFiles. It answers the repository, main's head and the release branch's head.
func aRepositoryWithARelease(t *testing.T, mainFiles, releaseFiles map[string]string) (string, string, string) {
t.Helper()
dir := t.TempDir()
gitInRepo(t, dir, "init", "--quiet", "-b", "main")
main := commitFiles(t, dir, mainFiles)
if releaseFiles == nil {
return dir, main, ""
}
gitInRepo(t, dir, "checkout", "--quiet", "-b", releaseBranch)
head := commitFiles(t, dir, releaseFiles)
gitInRepo(t, dir, "checkout", "--quiet", "main")
return dir, main, head
}
// aJudgeOf is a controller that judges as told: its gate passes, saying who it is and the word of the node-engine
// validator it vendors — the one it was built with.
func aJudgeOf(who string) map[string]string {
return map[string]string{
"go.mod": "module example.org/judge\n\ngo 1.22\n\nrequire github.com/novox/mesh-host v0.0.0\n",
"vendor/modules.txt": "# github.com/novox/mesh-host v0.0.0\n## explicit; go 1.22\ngithub.com/novox/mesh-host/validate\n",
"vendor/github.com/novox/mesh-host/validate/v.go": "package validate\n\nconst Word = \"the vendored validator\"\n",
"vendor/github.com/novox/mesh-host/internal/declaration/d.go": "package declaration\n",
"cmd/mesh-controller/main.go": `package main
import (
"fmt"
"os"
"github.com/novox/mesh-host/validate"
)
func main() {
if len(os.Args) == 2 && os.Args[1] == "merge-gate" {
fmt.Println("usage: merge-gate --facts <file> --store <postgres>")
os.Exit(2)
}
fmt.Printf("{\"verdict\":\"pass\",\"summary\":\"%s with %s\"}\n", "` + who + `", validate.Word)
}
`,
}
}
// aHostSaying is the node-engine's validator, saying a word.
func aHostSaying(word string) map[string]string {
return map[string]string{
"validate/v.go": "package validate\n\nconst Word = \"" + word + "\"\n",
"internal/declaration/d.go": "package declaration\n",
"validate/v_test.go": "package validate\n",
"internal/declaration/x_test.go": "package declaration\n",
}
}
// aRuntimeThatBuildsTheJudge puts a container runtime on PATH that runs every container's command on this machine,
// in the directory and with the environment the check gives it — the judge's build too — and a store and a bus that
// answer. It answers the file each `go` the check runs is written to.
func aRuntimeThatBuildsTheJudge(t *testing.T) string {
t.Helper()
if _, err := exec.LookPath("go"); err != nil {
t.Skip("no Go toolchain on this machine to build a judge with")
}
bus, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { bus.Close() })
go func() {
for {
c, err := bus.Accept()
if err != nil {
return
}
c.Close()
}
}()
dir := t.TempDir()
built := filepath.Join(dir, "built")
runtime := `#!/bin/sh
cmd="$1"; shift
case "$cmd" in
ps|rm|exec) exit 0 ;;
volume)
if [ -n "$VOLUMES" ]; then echo "$*" >> "$VOLUMES"; fi
exit 0 ;;
port) echo "` + bus.Addr().String() + `"; exit 0 ;;
run)
if [ -n "$RUNS" ]; then echo "$*" >> "$RUNS"; fi
workdir=""; detached=""
while [ $# -gt 0 ]; do
case "$1" in
-d) detached=1; shift ;;
--rm) shift ;;
--workdir) workdir="$2"; shift 2 ;;
--env) export "$2"; shift 2 ;;
--network|--volume|--user|--label|--name|-p|-e) shift 2 ;;
*) break ;;
esac
done
if [ -n "$detached" ]; then echo a-container; exit 0; fi
shift
if [ "$1" = go ]; then echo "GOCACHE=$GOCACHE GOENV=$GOENV $*" >> "` + built + `"; fi
if [ "$1" = go ] && [ "$2" = build ] && [ -n "$JUDGE_HOMES" ]; then echo "$HOME" >> "$JUDGE_HOMES"; fi
if [ "$1" = go ] && [ "$2" = test ] && [ -n "$REPLAYS_CANNOT_START" ]; then
echo "docker: Error response from daemon: invalid mount config for type bind" >&2; exit 125
fi
if [ "$1" = go ] && [ "$2" = test ] && [ -n "$REPLAYS" ]; then
echo "HOME=$HOME GOCACHE=$GOCACHE GOMODCACHE=$GOMODCACHE GOENV=$GOENV GOFLAGS=$GOFLAGS DIR=$workdir" >> "$REPLAYS"
fi
if [ "$1" = go ] && [ -n "$WARM_GO_CACHE" ] && [ -n "$GOCACHE" ] && [ -z "$(ls -A "$GOCACHE" 2>/dev/null)" ]; then
mkdir -p "$GOCACHE" && cp -R "$WARM_GO_CACHE/." "$GOCACHE/"
fi
cd "$workdir" && exec "$@" ;;
esac
exit 1
`
if err := os.WriteFile(filepath.Join(dir, "docker"), []byte(runtime), 0o755); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
t.Setenv("GOTOOLCHAIN", "local")
// An empty Go cache the check names is filled from a warm one before `go` runs (warm_go_cache_test.go).
t.Setenv("WARM_GO_CACHE", warmGoCache(t))
return built
}
// builds is how many times the check ran go to build a judge.
func builds(t *testing.T, file string) int {
t.Helper()
body, err := os.ReadFile(file)
if err != nil {
return 0
}
return strings.Count(string(body), "go build")
}
// theReleasedMesh is the core beside a release-branch check: a controller whose main and release branch judge
// differently, a node-engine with a validator of each, and a lab with no release branch.
func theReleasedMesh(t *testing.T) (map[string]Beside, string, string, string) {
t.Helper()
controller, _, controllerRelease := aRepositoryWithARelease(t, aJudgeOf("main's controller"),
aJudgeOf("the release branch's controller"))
host, _, hostRelease := aRepositoryWithARelease(t, aHostSaying("main's validator"),
aHostSaying("the release branch's validator"))
lab, labMain, _ := aRepositoryWithARelease(t, map[string]string{"README": "the lab"}, nil)
return map[string]Beside{
"mesh-controller": {Repository: controller, Ref: releaseBranch},
"mesh-controller-main": {Repository: controller, Ref: "main"},
"mesh-host": {Repository: host, Ref: releaseBranch, Else: "main"},
"mesh-lab": {Repository: lab, Ref: releaseBranch, Else: "main"},
}, controllerRelease, hostRelease, labMain
}
// aPullRequestInto is a repository whose pull request's head is cut from base, with a change.
func aPullRequestInto(t *testing.T, files map[string]string, base string, change map[string]string) (string, string) {
t.Helper()
repo, _, _ := aRepositoryWithARelease(t, files, map[string]string{"released": "r"})
gitInRepo(t, repo, "checkout", "--quiet", "-b", "change", base)
head := commitFiles(t, repo, change)
gitInRepo(t, repo, "checkout", "--quiet", "main")
return repo, head
}
func TestACataloguePullRequestIntoAReleaseBranchIsJudgedByThatBranchsController(t *testing.T) {
built := aRuntimeThatBuildsTheJudge(t)
registry := aFactsRegistry(t)
beside, controllerRelease, hostRelease, labMain := theReleasedMesh(t)
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
CheckScript: "true\n"}, releaseBranch, map[string]string{"modules/gitea/index.ts": "x"})
cache := t.TempDir()
check := func(id string) CheckVerdict {
t.Helper()
v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: catalogue, Ref: head, Owner: "novox",
Repo: "mesh-catalog", Number: 286, Base: releaseBranch, Release: releaseBranch, JudgeCache: cache,
Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Beside: beside, Toolchain: "golang"},
t.TempDir(), registry, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
return v
}
v := check("check-704-catalogue")
// The release branch's controller judged, with mesh-host's validator of the release branch.
if v.Gate == nil || v.Gate.Verdict != "pass" ||
!strings.HasSuffix(v.Gate.Summary, "the release branch's controller with the release branch's validator") {
t.Fatalf("the gate was not judged by the release branch's controller with its node-engine's validator: %+v\n%s",
v.Gate, v.Report)
}
// It names the release branch, the judge's commit, and each repository read beside it at its branch and commit.
mark := "[release " + releaseBranch + ", judged by its controller " + controllerRelease[:12] + "] "
if !strings.HasPrefix(v.Gate.Summary, mark) || !strings.HasPrefix(v.Repo.Summary, mark) {
t.Fatalf("the statuses do not name the release branch and its judge first:\n%s\n%s", v.Gate.Summary, v.Repo.Summary)
}
if v.Release != releaseBranch || v.JudgedBy != controllerRelease[:12] {
t.Fatalf("the verdict names release %q judged by %q, not %s by %s", v.Release, v.JudgedBy, releaseBranch,
controllerRelease[:12])
}
if v.BesideAt["mesh-host"] != releaseBranch || v.Beside["mesh-host"] != hostRelease[:12] {
t.Fatalf("mesh-host was read at %q (%s), not its release branch at %s", v.BesideAt["mesh-host"],
v.Beside["mesh-host"], hostRelease[:12])
}
// A repository with no release branch is read at its trunk, and says so.
if v.BesideAt["mesh-lab"] != "main" || v.Beside["mesh-lab"] != labMain[:12] {
t.Fatalf("the lab, with no release branch, was read at %q (%s), not main at %s", v.BesideAt["mesh-lab"],
v.Beside["mesh-lab"], labMain[:12])
}
if !strings.Contains(v.Report, "read beside it: ") || !strings.Contains(v.Report, "mesh-host "+releaseBranch+" at "+hostRelease[:12]) {
t.Fatalf("the report does not say what was read beside it:\n%s", v.Report)
}
// It says which validator judged: mesh-host's of the release branch, with its commit.
if want := "mesh-host's validator of " + releaseBranch + " at " + hostRelease[:12]; v.Validator != want ||
!strings.Contains(v.Report, want) {
t.Fatalf("the verdict says the validator was %q, not %q:\n%s", v.Validator, want, v.Report)
}
// Each repository read beside it is named with the branch and the commit read.
for dir := range beside {
if v.BesideAt[dir] == "" || v.Beside[dir] == "" ||
!strings.Contains(v.Report, dir+" "+v.BesideAt[dir]+" at "+v.Beside[dir]) {
t.Fatalf("%s is not named with its branch and commit: at %q, commit %q\n%s", dir, v.BesideAt[dir],
v.Beside[dir], v.Report)
}
}
if n := builds(t, built); n != 1 {
t.Fatalf("the judge was built %d times for one check", n)
}
// **The judge is built with the judge cache's Go cache** (ADR 0358 §6), which no check's container mounts, and no
// Go environment file.
said, _ := os.ReadFile(built)
if !strings.Contains(string(said), "GOCACHE="+filepath.Join(cache, "go-cache")+" GOENV=off go build") {
t.Fatalf("the release judge was not built with the judge cache's Go cache:\n%s", said)
}
// **Clause 5**: the head was merged onto the release branch's head, which the verdict names as its trunk commit,
// so the release branch's move puts it out of date as a trunk's move does (ADR 0323 §4, the forge's holder).
if releaseHead := gitInRepo(t, catalogue, "rev-parse", releaseBranch); v.Trunk != releaseHead[:12] {
t.Fatalf("the verdict names trunk %q, not the release branch's head %s", v.Trunk, releaseHead[:12])
}
// **Built once per release-branch head**: a second check while that head stands takes the judge kept.
v = check("check-704-catalogue-again")
if !strings.HasSuffix(v.Gate.Summary, "the release branch's controller with the release branch's validator") {
t.Fatalf("the kept judge did not judge: %+v\n%s", v.Gate, v.Report)
}
if n := builds(t, built); n != 1 {
t.Fatalf("the judge of the same release-branch head was built again: %d builds", n)
}
}
func TestAControllerPullRequestIntoAReleaseBranchReadsTheNodeEngineAtThatBranch(t *testing.T) {
aRuntimeThatBuildsTheJudge(t)
registry := aFactsRegistry(t)
beside, _, hostRelease, _ := theReleasedMesh(t)
delete(beside, "mesh-controller")
controller, head := aPullRequestInto(t, aJudgeOf("the change's controller"), releaseBranch,
map[string]string{"cmd/x.txt": "x"})
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-controller", Repository: controller, Ref: head,
Owner: "novox", Repo: "mesh-controller", Number: 315, Base: releaseBranch, Release: releaseBranch,
JudgeCache: t.TempDir(), Paths: []string{"cmd/x.txt"}, Modules: []string{"mesh-controller"}, Judge: "self",
Beside: beside, Toolchain: "golang"}, t.TempDir(), registry, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if v.Gate == nil || !strings.HasSuffix(v.Gate.Summary, "the change's controller with the vendored validator") {
t.Fatalf("a controller change into a release branch did not judge itself: %+v\n%s", v.Gate, v.Report)
}
if v.BesideAt["mesh-host"] != releaseBranch || v.Beside["mesh-host"] != hostRelease[:12] {
t.Fatalf("mesh-host was read beside it at %q (%s), not at the release branch's %s", v.BesideAt["mesh-host"],
v.Beside["mesh-host"], hostRelease[:12])
}
if v.Release != releaseBranch || v.JudgedBy == "" || v.JudgedBy != v.Merge[:12] {
t.Fatalf("the verdict names release %q judged by %q, not its own merge %s", v.Release, v.JudgedBy, v.Merge)
}
}
func TestAPullRequestIntoTheTrunkIsJudgedAsBefore(t *testing.T) {
aRuntimeThatBuildsTheJudge(t)
registry := aFactsRegistry(t)
controller, controllerMain, _ := aRepositoryWithARelease(t, aJudgeOf("main's controller"),
aJudgeOf("the release branch's controller"))
host, hostMain, _ := aRepositoryWithARelease(t, aHostSaying("main's validator"),
aHostSaying("the release branch's validator"))
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
CheckScript: "true\n"}, "main", map[string]string{"modules/gitea/index.ts": "x"})
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-trunk", Repository: catalogue, Ref: head,
Owner: "novox", Repo: "mesh-catalog", Number: 1, Base: "main", JudgeCache: t.TempDir(),
Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Toolchain: "golang",
Beside: map[string]Beside{"mesh-controller": {Repository: controller, Ref: controllerMain},
"mesh-host": {Repository: host, Ref: hostMain}}}, t.TempDir(), registry, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if v.Gate == nil || v.Gate.Summary != "main's controller with the vendored validator" {
t.Fatalf("a trunk pull request was not judged by the controller beside it as before: %+v\n%s", v.Gate, v.Report)
}
if v.Release != "" || v.JudgedBy != "" || v.BesideAt != nil || strings.Contains(v.Report, "release branch") {
t.Fatalf("a trunk pull request's verdict speaks of a release branch: %q %q %v", v.Release, v.JudgedBy, v.BesideAt)
}
if v.Beside["mesh-host"] != hostMain[:12] {
t.Fatalf("mesh-host was read at %s, not the commit asked %s", v.Beside["mesh-host"], hostMain[:12])
}
}
// **Without mesh-host's release branch, the release branch's controller keeps the validator it vendors** (ADR 0358
// §2), and the verdict says so.
func TestWithoutTheNodeEnginesReleaseBranchTheJudgeKeepsItsVendoredValidator(t *testing.T) {
aRuntimeThatBuildsTheJudge(t)
registry := aFactsRegistry(t)
beside, controllerRelease, _, _ := theReleasedMesh(t)
host, hostMain, _ := aRepositoryWithARelease(t, aHostSaying("main's validator"), nil)
beside["mesh-host"] = Beside{Repository: host, Ref: releaseBranch, Else: "main"}
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
CheckScript: "true\n"}, releaseBranch, map[string]string{"modules/gitea/index.ts": "x"})
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-vendored", Repository: catalogue, Ref: head,
Owner: "novox", Repo: "mesh-catalog", Number: 286, Base: releaseBranch, Release: releaseBranch,
JudgeCache: t.TempDir(), Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Beside: beside,
Toolchain: "golang"}, t.TempDir(), registry, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if v.Gate == nil || !strings.HasSuffix(v.Gate.Summary, "the release branch's controller with the vendored validator") {
t.Fatalf("without mesh-host's release branch the judge did not keep its vendored validator: %+v\n%s", v.Gate, v.Report)
}
if v.JudgedBy != controllerRelease[:12] || v.BesideAt["mesh-host"] != "main" || v.Beside["mesh-host"] != hostMain[:12] {
t.Fatalf("judged by %q, mesh-host read at %q (%s)", v.JudgedBy, v.BesideAt["mesh-host"], v.Beside["mesh-host"])
}
if !strings.Contains(v.Validator, "controller vendors") || !strings.Contains(v.Report, v.Validator) {
t.Fatalf("the verdict does not say the vendored validator judged: %q\n%s", v.Validator, v.Report)
}
}
// **Clause 7** (ADR 0358 §7): a release branch's own pull request into its trunk is checked as any pull request into a
// trunk — by the controller beside it as the mesh runs it, the repositories at what a trunk check reads — and its
// verdict names no release branch.
func TestAReleaseBranchsPullRequestIntoItsTrunkIsJudgedByTheControllerTheMeshRuns(t *testing.T) {
aRuntimeThatBuildsTheJudge(t)
registry := aFactsRegistry(t)
controller, controllerMain, _ := aRepositoryWithARelease(t, aJudgeOf("the controller the mesh runs"),
aJudgeOf("the release branch's controller"))
host, hostMain, _ := aRepositoryWithARelease(t, aHostSaying("main's validator"),
aHostSaying("the release branch's validator"))
catalogue, _, releaseHead := aRepositoryWithARelease(t, map[string]string{
"modules/gitea/module.json": `{"module":"gitea"}`, CheckScript: "true\n"},
map[string]string{"modules/gitea/index.ts": "released"})
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-release-into-trunk", Repository: catalogue,
Ref: releaseHead, Owner: "novox", Repo: "mesh-catalog", Number: 300, Base: "main", JudgeCache: t.TempDir(),
Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Toolchain: "golang",
Beside: map[string]Beside{"mesh-controller": {Repository: controller, Ref: controllerMain},
"mesh-host": {Repository: host, Ref: hostMain}}}, t.TempDir(), registry, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if v.Gate == nil || v.Gate.Summary != "the controller the mesh runs with the vendored validator" {
t.Fatalf("the release branch's pull request into its trunk was not judged by the controller the mesh runs: %+v\n%s",
v.Gate, v.Report)
}
if v.Release != "" || v.JudgedBy != "" || v.BesideAt != nil || v.Validator != "" ||
strings.Contains(v.Report, "release branch") || strings.Contains(v.Gate.Summary, "[release") {
t.Fatalf("its verdict names a release branch: %q %q %v %q", v.Release, v.JudgedBy, v.BesideAt, v.Validator)
}
if v.Beside["mesh-host"] != hostMain[:12] || v.Beside["mesh-controller"] != controllerMain[:12] {
t.Fatalf("beside it were read %v, not the trunk's %s and %s", v.Beside, controllerMain[:12], hostMain[:12])
}
}
// **mesh-lab is read at its trunk, always** (ADR 0358): its replays run with the container runtime's socket, so a
// branch of the release's name in mesh-lab — which anyone may create — is never what a check reads.
func TestAReleaseBranchCheckReadsMeshLabAtItsTrunkEvenWhenItHasThatBranch(t *testing.T) {
aRuntimeThatBuildsTheJudge(t)
registry := aFactsRegistry(t)
beside, _, _, _ := theReleasedMesh(t)
lab, labMain, labRelease := aRepositoryWithARelease(t, map[string]string{"README": "the lab"},
map[string]string{"README": "a branch anyone could push"})
beside["mesh-lab"] = Beside{Repository: lab, Ref: releaseBranch, Else: "main"}
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
CheckScript: "true\n"}, releaseBranch, map[string]string{"modules/gitea/index.ts": "x"})
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-lab", Repository: catalogue, Ref: head,
Owner: "novox", Repo: "mesh-catalog", Number: 286, Base: releaseBranch, Release: releaseBranch,
JudgeCache: t.TempDir(), Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Beside: beside,
Toolchain: "golang"}, t.TempDir(), registry, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if v.BesideAt["mesh-lab"] != "main" || v.Beside["mesh-lab"] != labMain[:12] || v.Beside["mesh-lab"] == labRelease[:12] {
t.Fatalf("mesh-lab was read at %q (%s), not its trunk main at %s", v.BesideAt["mesh-lab"], v.Beside["mesh-lab"],
labMain[:12])
}
}
// aPlantedWorkspace is a workspace a pull request's own script has written into, as it can: a git configuration in its
// HOME whose fsmonitor hook says each directory git runs in, a git configuration under XDG_CONFIG_HOME doing the same,
// and a Go environment file that would hand the judge's build a tool of its own. It answers the workspace and the
// file the hook writes to.
func aPlantedWorkspace(t *testing.T) (string, string) {
t.Helper()
workspace, logs := t.TempDir(), t.TempDir()
ran := filepath.Join(logs, "hook-ran")
hook := filepath.Join(workspace, "hook.sh")
files := map[string]string{
hook: "#!/bin/sh\necho \"$PWD\" >> " + ran + "\n",
filepath.Join(workspace, ".gitconfig"): "[core]\n\tfsmonitor = " + hook + "\n",
filepath.Join(workspace, ".config", "git", "config"): "[core]\n\tfsmonitor = " + hook + "\n",
filepath.Join(workspace, ".config", "go", "env"): "GOFLAGS=-toolexec=" + hook + "\n",
}
for name, body := range files {
if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(name, []byte(body), 0o755); err != nil {
t.Fatal(err)
}
}
return workspace, ran
}
// judgeRanTheHook says whether the planted hook ran in a judge's tree: the controller beside the check, or the
// change's own when it judges itself.
func judgeRanTheHook(t *testing.T, ran string) string {
t.Helper()
body, _ := os.ReadFile(ran)
for _, line := range strings.Split(string(body), "\n") {
if strings.HasSuffix(line, "/check/mesh-controller") {
return string(body)
}
}
return ""
}
// **Every judge is built where no pull request reaches** (issue 482, ADR 0358 review): a git configuration with a
// hook, and a Go environment file, planted in the workspace by an earlier pull request's script, do not run during
// the release judge's build — while the pull request's own script, in the workspace, still meets them.
func TestAHookPlantedInTheWorkspaceDoesNotRunDuringTheReleaseJudgesBuild(t *testing.T) {
aRuntimeThatBuildsTheJudge(t)
homes := filepath.Join(t.TempDir(), "homes")
t.Setenv("JUDGE_HOMES", homes)
registry := aFactsRegistry(t)
workspace, ran := aPlantedWorkspace(t)
beside, _, _, _ := theReleasedMesh(t)
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
CheckScript: "git status >/dev/null 2>&1; true\n"}, releaseBranch, map[string]string{"modules/gitea/index.ts": "x"})
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-planted", Repository: catalogue, Ref: head,
Owner: "novox", Repo: "mesh-catalog", Number: 286, Base: releaseBranch, Release: releaseBranch,
JudgeCache: t.TempDir(), Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Beside: beside,
Toolchain: "golang"}, workspace, registry, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if v.Gate == nil || v.Gate.Verdict != "pass" {
t.Fatalf("the check did not judge: %+v\n%s", v.Gate, v.Report)
}
if body, _ := os.ReadFile(ran); !strings.Contains(string(body), "/check/mesh-catalog") {
t.Fatalf("the planted hook never ran, so this test proves nothing: %q", body)
}
if said := judgeRanTheHook(t, ran); said != "" {
t.Fatalf("the hook planted in the workspace ran during the judge's build:\n%s", said)
}
if body, _ := os.ReadFile(homes); len(body) == 0 || strings.Contains(string(body), workspace) {
t.Fatalf("the judge was built with a HOME in the workspace: %q", body)
}
}
// The same for the judge built with a node-engine change's own validator, into a release branch (ADR 0358 §2).
func TestAHookPlantedInTheWorkspaceDoesNotRunDuringTheValidatorJudgesBuild(t *testing.T) {
aRuntimeThatBuildsTheJudge(t)
homes := filepath.Join(t.TempDir(), "homes")
t.Setenv("JUDGE_HOMES", homes)
registry := aFactsRegistry(t)
workspace, ran := aPlantedWorkspace(t)
beside, _, _, _ := theReleasedMesh(t)
delete(beside, "mesh-host")
files := aHostSaying("the release branch's validator")
files["module.json"] = `{"module":"mesh-host"}`
files[CheckScript] = "git status >/dev/null 2>&1; true\n"
host, head := aPullRequestInto(t, files, releaseBranch, aHostSaying("this change's validator"))
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-planted-validator", Repository: host, Ref: head,
Owner: "novox", Repo: "mesh-host", Number: 111, Base: releaseBranch, Release: releaseBranch, Judge: "validator",
JudgeCache: t.TempDir(), Paths: []string{"validate/v.go"}, Modules: []string{"mesh-host"}, Beside: beside,
Toolchain: "golang"}, workspace, registry, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if v.Gate == nil || !strings.HasSuffix(v.Gate.Summary, "the release branch's controller with this change's validator") {
t.Fatalf("a node-engine change into a release branch was not judged by that branch's controller with its own "+
"validator: %+v\n%s", v.Gate, v.Report)
}
if body, _ := os.ReadFile(ran); !strings.Contains(string(body), "/check/mesh-host") {
t.Fatalf("the planted hook never ran, so this test proves nothing: %q", body)
}
if said := judgeRanTheHook(t, ran); said != "" {
t.Fatalf("the hook planted in the workspace ran during a judge's build:\n%s", said)
}
if body, _ := os.ReadFile(homes); strings.Count(string(body), "\n") != 2 || strings.Contains(string(body), workspace) {
t.Fatalf("the judges were not both built with a HOME outside the workspace: %q", body)
}
}
// **mesh-lab is never read at a release branch**, asked with a fallback or without (ADR 0358).
func TestTheBuildSeatNeverReadsMeshLabAtAReleaseBranch(t *testing.T) {
aRuntimeThatBuildsTheJudge(t)
registry := aFactsRegistry(t)
lab, labMain, _ := aRepositoryWithARelease(t, map[string]string{"README": "the lab"},
map[string]string{"README": "a branch anyone could push"})
for _, asked := range []Beside{{Repository: lab, Ref: releaseBranch}, {Repository: lab, Ref: releaseBranch, Else: "main"}} {
beside, _, _, _ := theReleasedMesh(t)
beside["mesh-lab"] = asked
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
CheckScript: "true\n"}, releaseBranch, map[string]string{"modules/gitea/index.ts": "x"})
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-lab-asked", Repository: catalogue, Ref: head,
Owner: "novox", Repo: "mesh-catalog", Number: 286, Base: releaseBranch, Release: releaseBranch,
JudgeCache: t.TempDir(), Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Beside: beside,
Toolchain: "golang"}, t.TempDir(), registry, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if v.BesideAt["mesh-lab"] != "main" || v.Beside["mesh-lab"] != labMain[:12] {
t.Fatalf("mesh-lab asked as %+v was read at %q (%s), not main at %s", asked, v.BesideAt["mesh-lab"],
v.Beside["mesh-lab"], labMain[:12])
}
}
}
// **Judges are built warm and where no pull request reaches** (ADR 0358 §6): every judge build uses the judge cache's
// one Go cache, kept between checks; its home is made fresh under the judge cache and removed after; nothing is left in
// a temporary directory of the host; and no container a check runs for the pull request mounts the judge cache.
func TestJudgesAreBuiltWarmFromAJudgeCacheNoCheckContainerMounts(t *testing.T) {
built := aRuntimeThatBuildsTheJudge(t)
// Not filled from the test's warm cache: what the first judge's build leaves in the judge cache is what is proved.
t.Setenv("WARM_GO_CACHE", "")
runs := filepath.Join(t.TempDir(), "runs")
t.Setenv("RUNS", runs)
tmp := t.TempDir()
t.Setenv("TMPDIR", tmp)
registry := aFactsRegistry(t)
cache := t.TempDir()
controller, controllerMain, _ := aRepositoryWithARelease(t, aJudgeOf("the controller the mesh runs"), nil)
beside := map[string]Beside{"mesh-controller": {Repository: controller, Ref: controllerMain}}
for i := range 2 {
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
CheckScript: "true\n"}, "main", map[string]string{"modules/gitea/index.ts": fmt.Sprint(i)})
v, err := Check(t.Context(), Command, CheckSpec{ID: fmt.Sprintf("check-704-warm-%d", i), Repository: catalogue,
Ref: head, Owner: "novox", Repo: "mesh-catalog", Number: 1, Base: "main", JudgeCache: cache,
Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Toolchain: "golang", Beside: beside},
t.TempDir(), registry, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if v.Gate == nil || v.Gate.Summary != "the controller the mesh runs with the vendored validator" {
t.Fatalf("check %d was not judged: %+v\n%s", i, v.Gate, v.Report)
}
if i == 0 {
if entries, _ := os.ReadDir(filepath.Join(cache, "go-cache")); len(entries) == 0 {
t.Fatal("the judge's build left nothing in the judge cache's Go cache to build the next judge warm")
}
}
}
// Both judges built with the one Go cache of the judge cache.
said, _ := os.ReadFile(built)
if n := strings.Count(string(said), "GOCACHE="+filepath.Join(cache, "go-cache")+" GOENV=off go build"); n != 2 {
t.Fatalf("%d of 2 judge builds used the judge cache's Go cache:\n%s", n, said)
}
// Nothing left behind: no home under the judge cache, nothing in the host's temporary directory.
if left, _ := os.ReadDir(filepath.Join(cache, "homes")); len(left) != 0 {
t.Fatalf("judge builds left %d home(s) under the judge cache", len(left))
}
if left, _ := os.ReadDir(tmp); len(left) != 0 {
t.Fatalf("a check left %d entr(ies) in the temporary directory, e.g. %s", len(left), left[0].Name())
}
// Only a judge build mounts the judge cache: no container the check runs for the pull request does.
body, _ := os.ReadFile(runs)
judgeBuilds := 0
for _, line := range strings.Split(strings.TrimSpace(string(body)), "\n") {
judging := strings.Contains(line, " go build ")
if judging {
judgeBuilds++
if !strings.Contains(line, "--volume "+filepath.Join(cache, "go-cache")+":") {
t.Fatalf("a judge build did not mount the judge cache's Go cache: %s", line)
}
continue
}
if strings.Contains(line, cache) {
t.Fatalf("a container the check runs for the pull request names the judge cache: %s", line)
}
}
if judgeBuilds != 2 {
t.Fatalf("%d judge builds were run, not 2", judgeBuilds)
}
}
// Without a judge cache, a judge is built cold under the check's own root — never in a temporary directory of the
// host — and a judge cache inside the workspace is refused.
func TestWithoutAJudgeCacheAJudgeIsBuiltColdUnderTheChecksRoot(t *testing.T) {
built := aRuntimeThatBuildsTheJudge(t)
tmp := t.TempDir()
t.Setenv("TMPDIR", tmp)
registry := aFactsRegistry(t)
controller, controllerMain, _ := aRepositoryWithARelease(t, aJudgeOf("the controller the mesh runs"), nil)
beside := map[string]Beside{"mesh-controller": {Repository: controller, Ref: controllerMain}}
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
CheckScript: "true\n"}, "main", map[string]string{"modules/gitea/index.ts": "x"})
workspace := t.TempDir()
spec := CheckSpec{ID: "check-704-cold", Repository: catalogue, Ref: head, Owner: "novox", Repo: "mesh-catalog",
Number: 1, Base: "main", Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"},
Toolchain: "golang", Beside: beside}
v, err := Check(t.Context(), Command, spec, workspace, registry, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if v.Gate == nil || v.Gate.Verdict != "pass" {
t.Fatalf("the check was not judged: %+v\n%s", v.Gate, v.Report)
}
said, _ := os.ReadFile(built)
if !strings.Contains(string(said), "GOCACHE="+filepath.Join(workspace, "check", "judge-go-cache")+" GOENV=off go build") {
t.Fatalf("without a judge cache the judge was not built under the check's root:\n%s", said)
}
if left, _ := os.ReadDir(tmp); len(left) != 0 {
t.Fatalf("a check left %d entr(ies) in the temporary directory", len(left))
}
spec.JudgeCache = filepath.Join(workspace, "judge-cache")
if _, err := Check(t.Context(), Command, spec, workspace, registry, GitCredential{}, nil); err == nil ||
!strings.Contains(err.Error(), "inside the workspace") {
t.Fatalf("a judge cache inside the workspace was taken: %v", err)
}
}
// aLabWithReplays is mesh-lab with replays: a Go module of its own whose one test passes, needing nothing fetched.
func aLabWithReplays(t *testing.T) (string, string) {
t.Helper()
lab, labMain, _ := aRepositoryWithARelease(t, map[string]string{
"replays/go.mod": "module example.org/replays\n\ngo 1.22\n",
"replays/replay_test.go": "package replays\n\nimport \"testing\"\n\nfunc TestAReplay(t *testing.T) {}\n",
}, nil)
return lab, labMain
}
// **mesh-lab's replays run where no pull request reaches** (novox/hq issue 709, ADR 0358 §6): they hold the container
// runtime's socket, so a Go environment file with a -toolexec hook and a Go cache and module cache planted in the
// workspace by a pull request's script are neither run nor read — the replays run from a copy of mesh-lab outside
// the workspace with a fresh home and the judge cache's caches, all removed or kept outside every check container.
func TestTheReplaysRunWhereNoPullRequestReaches(t *testing.T) {
aRuntimeThatBuildsTheJudge(t)
replays := filepath.Join(t.TempDir(), "replays")
t.Setenv("REPLAYS", replays)
registry := aFactsRegistry(t)
workspace, ran := aPlantedWorkspace(t)
for _, poisoned := range []string{filepath.Join(workspace, "go-cache", "README"),
filepath.Join(workspace, "go-modules", "cache", "download", "planted")} {
if err := os.MkdirAll(filepath.Dir(poisoned), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(poisoned, []byte("planted by a pull request"), 0o644); err != nil {
t.Fatal(err)
}
}
cache := t.TempDir()
controller, controllerMain, _ := aRepositoryWithARelease(t, aJudgeOf("the controller the mesh runs"), nil)
lab, labMain := aLabWithReplays(t)
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
CheckScript: "true\n"}, "main", map[string]string{"modules/gitea/index.ts": "x"})
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-709", Repository: catalogue, Ref: head, Owner: "novox",
Repo: "mesh-catalog", Number: 1, Base: "main", JudgeCache: cache, Paths: []string{"modules/gitea/index.ts"},
Modules: []string{"gitea"}, Toolchain: "golang", Beside: map[string]Beside{
"mesh-controller": {Repository: controller, Ref: controllerMain}, "mesh-lab": {Repository: lab, Ref: labMain}}},
workspace, registry, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if v.Gate == nil || v.Gate.Verdict != "pass" || !strings.Contains(v.Report, "--- the replays") {
t.Fatalf("the replays did not run and pass: %+v\n%s", v.Gate, v.Report)
}
body, _ := os.ReadFile(replays)
said := strings.TrimSpace(string(body))
if said == "" || strings.Count(said, "\n") != 0 {
t.Fatalf("the replays ran %q, not once", said)
}
for _, want := range []string{"GOCACHE=" + filepath.Join(cache, "go-cache") + " ",
"GOMODCACHE=" + filepath.Join(cache, "go-modules") + " ", "GOENV=off ", "GOFLAGS=-buildvcs=false ",
"HOME=" + filepath.Join(cache, "homes", "check-709-replays", "home") + " ",
"DIR=" + filepath.Join(cache, "homes", "check-709-replays", "mesh-lab", "replays")} {
if !strings.Contains(said, want) {
t.Fatalf("the replays did not run with %q: %s", want, said)
}
}
if strings.Contains(said, workspace) {
t.Fatalf("the replays ran with something of the workspace: %s", said)
}
if body, _ := os.ReadFile(ran); strings.Contains(string(body), "replays") || strings.Contains(string(body), "mesh-lab") {
t.Fatalf("a hook planted in the workspace ran during the replays:\n%s", body)
}
if left, _ := os.ReadDir(filepath.Join(cache, "homes")); len(left) != 0 {
t.Fatalf("the replays left %d entr(ies) under the judge cache's homes", len(left))
}
}
// aSelfJudgingControllerThatWritesIntoTheReplays is a controller pull request whose own judge, while the gate runs it,
// writes a test into the replays beside the check — the reviewer's case for issue 709: code of the pull request's,
// run before the replays, reaching what they run.
func aSelfJudgingControllerThatWritesIntoTheReplays(t *testing.T, marker string) (string, string) {
t.Helper()
files := aJudgeOf("the change's controller")
evil := "package replays\n\nimport (\n\t\"os\"\n\t\"testing\"\n)\n\nfunc TestEvil(t *testing.T) {\n\t_ = os.WriteFile(" +
fmt.Sprintf("%q", marker) + ", []byte(\"ran\"), 0o644)\n}\n"
files["cmd/mesh-controller/main.go"] = strings.Replace(files["cmd/mesh-controller/main.go"], "func main() {",
"func main() {\n\tif len(os.Args) > 2 && os.Args[1] == \"merge-gate\" {\n\t\t_ = os.WriteFile(os.Getenv(\"MESH_CHECK_BESIDE\")+"+
"\"/mesh-lab/replays/evil_test.go\", []byte("+fmt.Sprintf("%q", evil)+"), 0o644)\n\t\t_ = os.WriteFile("+
fmt.Sprintf("%q", marker+".wrote")+", nil, 0o644)\n\t}", 1)
return aPullRequestInto(t, files, "main", map[string]string{"cmd/x.txt": "x"})
}
// **The replays never read what a pull request wrote** (issue 709, reviews 12303 and 12308): with a judge cache they
// run from a copy under it, and a self-judging controller pull request writing a test into the replays beside the
// check, a planted GOFLAGS hook and a poisoned cache entry in the workspace neither run nor are read. Without one the
// replays are refused — the gate an error that says why, never a pass, never the workspace — they never start, and
// nothing is made on the host.
func TestTheReplaysNeverReadWhatAPullRequestWroteAndNeedAJudgeCache(t *testing.T) {
for _, withCache := range []bool{true, false} {
t.Run(fmt.Sprintf("judge cache %v", withCache), func(t *testing.T) {
aRuntimeThatBuildsTheJudge(t)
replays, volumes := filepath.Join(t.TempDir(), "replays"), filepath.Join(t.TempDir(), "volumes")
t.Setenv("REPLAYS", replays)
t.Setenv("VOLUMES", volumes)
tmp := t.TempDir()
t.Setenv("TMPDIR", tmp)
registry := aFactsRegistry(t)
workspace, ran := aPlantedWorkspace(t)
if err := os.MkdirAll(filepath.Join(workspace, "go-cache"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(workspace, "go-cache", "README"), []byte("planted"), 0o644); err != nil {
t.Fatal(err)
}
evilRan := filepath.Join(t.TempDir(), "evil-ran")
controller, head := aSelfJudgingControllerThatWritesIntoTheReplays(t, evilRan)
lab, labMain := aLabWithReplays(t)
cache := ""
if withCache {
cache = t.TempDir()
}
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-709-self", Repository: controller, Ref: head,
Owner: "novox", Repo: "mesh-controller", Number: 330, Base: "main", Judge: "self", JudgeCache: cache,
Paths: []string{"cmd/x.txt"}, Modules: []string{"mesh-controller"}, Toolchain: "golang",
Beside: map[string]Beside{"mesh-lab": {Repository: lab, Ref: labMain}}}, workspace, registry,
GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(evilRan + ".wrote"); err != nil {
t.Fatalf("the self-judging pull request did not write into the replays beside it, so this proves nothing: %v", err)
}
if _, err := os.Stat(evilRan); err == nil {
t.Fatal("the test the pull request wrote into the replays ran with the socket")
}
if body, _ := os.ReadFile(ran); strings.Contains(string(body), "replays") || strings.Contains(string(body), "mesh-lab") {
t.Fatalf("a hook planted in the workspace ran during the replays:\n%s", body)
}
body, _ := os.ReadFile(replays)
said := strings.TrimSpace(string(body))
if vols, _ := os.ReadFile(volumes); len(vols) != 0 {
t.Fatalf("the check made a container volume: %q", vols)
}
if left, _ := os.ReadDir(tmp); len(left) != 0 {
t.Fatalf("a check left %d entr(ies) in the temporary directory", len(left))
}
if !withCache {
if v.Gate == nil || v.Gate.Verdict != "error" || !strings.Contains(v.Gate.Summary, replaysNeedTheJudgeCache) ||
v.Verdict != "error" {
t.Fatalf("without a judge cache the gate said %+v, not an error saying the replays need it", v.Gate)
}
if said != "" {
t.Fatalf("without a judge cache the replays started: %s", said)
}
return
}
if v.Gate == nil || v.Gate.Verdict != "pass" || !strings.Contains(v.Report, "--- the replays") {
t.Fatalf("the replays did not run and pass: %+v\n%s", v.Gate, v.Report)
}
base := filepath.Join(cache, "homes", "check-709-self-replays")
if said == "" || strings.Contains(said, workspace) || !strings.Contains(said, "GOENV=off ") ||
!strings.Contains(said, "DIR="+filepath.Join(base, "mesh-lab", "replays")) {
t.Fatalf("the replays did not run from the copy under %s, apart from the workspace: %q", base, said)
}
if _, err := os.Stat(base); err == nil {
t.Fatalf("the replays' place %s was left after the check", base)
}
})
}
}
// **Replays the container runtime could not start are the check's error, never the change's failure**, said in the
// build's log (review 12308); and a check whose id cannot name a place in the judge cache is refused, never given a
// shared one.
func TestReplaysThatCannotStartAreAnErrorAndAnUnsafeIDIsRefused(t *testing.T) {
aRuntimeThatBuildsTheJudge(t)
t.Setenv("REPLAYS_CANNOT_START", "1")
registry := aFactsRegistry(t)
controller, controllerMain, _ := aRepositoryWithARelease(t, aJudgeOf("the controller the mesh runs"), nil)
lab, labMain := aLabWithReplays(t)
catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`,
CheckScript: "true\n"}, "main", map[string]string{"modules/gitea/index.ts": "x"})
spec := CheckSpec{ID: "check-709-start", Repository: catalogue, Ref: head, Owner: "novox", Repo: "mesh-catalog",
Number: 1, Base: "main", JudgeCache: t.TempDir(), Paths: []string{"modules/gitea/index.ts"},
Modules: []string{"gitea"}, Toolchain: "golang", Beside: map[string]Beside{
"mesh-controller": {Repository: controller, Ref: controllerMain}, "mesh-lab": {Repository: lab, Ref: labMain}}}
var logged []string
v, err := Check(t.Context(), Command, spec, t.TempDir(), registry, GitCredential{}, func(step, message string) {
logged = append(logged, message)
})
if err != nil {
t.Fatal(err)
}
if v.Gate == nil || v.Gate.Verdict != "error" || !strings.Contains(v.Gate.Summary, "could not be started") ||
!strings.Contains(v.Gate.Summary, "invalid mount config") {
t.Fatalf("replays that could not start were said %+v, not an error naming why", v.Gate)
}
if !strings.Contains(strings.Join(logged, "\n"), "the replays could not be started") {
t.Fatalf("the build's log does not say what failed:\n%s", strings.Join(logged, "\n"))
}
spec.ID = "Check 1/../../x"
if _, err := Check(t.Context(), Command, spec, t.TempDir(), registry, GitCredential{}, nil); err == nil ||
!strings.Contains(err.Error(), "cannot name its place in the judge cache") {
t.Fatalf("a check with an unsafe id was taken: %v", err)
}
}
+125
View File
@@ -0,0 +1,125 @@
package builder
import (
"context"
"errors"
"path/filepath"
"strings"
"testing"
)
// novox/hq issue 713: after ADR 0350 the walk of novox/mesh-controller#331 built mesh-controller, build-agent and
// route-proxy again, each "built: no check of novox/mesh-controller#331 built it". The check had built none: its
// build log said "built for the merge: 0 of 3 module(s)", mesh-controller's go bundle failing with "go.mod file not
// found". The build seat's holder runs in a container sharing only its workspace with the host, and a build's
// toolchain container is run by the host's container engine, which mounts a path as the host names it. The check
// built in a temporary directory of the holder's own (issue 703's copy), which the engine mounted as an empty one.
// aHostsEngine runs as the host's container engine does for a holder in a container: a mount of a path outside the
// workspace the holder shares with the host is an empty directory, and a Go toolchain finds no go.mod in it.
type aHostsEngine struct {
onTrunk
shared string
}
func (h aHostsEngine) run(ctx context.Context, dir, name string, args ...string) (string, error) {
if name == "docker" && len(args) > 0 && args[0] == "run" {
for i, a := range args {
if a != "--volume" || i+1 >= len(args) {
continue
}
from, _, _ := strings.Cut(args[i+1], ":")
if !withinDir(h.shared, from) {
return "", errors.New("go: go.mod file not found in current directory or any parent directory")
}
}
}
return h.onTrunk.run(ctx, dir, name, args...)
}
// aGoModule is a module of one Go bundle built from a repository's root.
func aGoModule(module string) map[string]string {
files := aSharedRepository("one", "1")
files[ManifestName] = `{"module":"` + module + `","version":"1","build":{"artifacts":[
{"name":"server","kind":"bundle","language":"go","system":"arch","from":"cmd/ctl","binary":"ctl"}]},
"resources":[{"id":"server","type":"process","name":"` + module + `","artifact":"server","run":["./ctl"]}]}`
return files
}
// A controller change whose walk moves mesh-controller, build-agent and route-proxy (#331's) has its check build all
// three, in the workspace the host's engine sees, and the walk of its merge takes each from the check by its key.
func TestReplay713TheWalkOfAControllerChangeTakesAllThreeFromItsCheck(t *testing.T) {
withVersion(t, "builder-1")
pull := PullRef{Owner: "novox", Repo: "mesh-controller", Number: 331}
held := map[string]string{"mesh-tools-go/build": "registry.invalid/mesh-tools-go/build@sha256:" + strings.Repeat("b", 64)}
repositories := map[string]string{"build-agent": "https://forge.invalid/mesh-catalog-build-agent.git",
"route-proxy": "https://forge.invalid/mesh-catalog-route-proxy.git"}
checked := func(shared string) (*recorded, aHostsEngine) {
r := &recorded{tree: "tree-of-the-merge", contents: aGoModule("mesh-controller"),
secondary: map[string]map[string]string{}}
for module, repository := range repositories {
r.secondary[repository] = aGoModule(module)
}
return r, aHostsEngine{onTrunk: onTrunk{recorded: r}, shared: shared}
}
spec := CheckSpec{ID: "build-331", Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number,
Builds: []CheckBuildSpec{{Module: "mesh-controller", Merged: true, Tier: 0},
{Module: "build-agent", Repository: repositories["build-agent"], Ref: "main", Tier: 1},
{Module: "route-proxy", Repository: repositories["route-proxy"], Ref: "main", Tier: 2}}}
judged := CheckVerdict{Head: "98e856874913", Trunk: "c975dbf7b053", Merge: "2d22279b1d07"}
passed := CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "pass"}}
// As it ran: the builds made in the holder's own temporary directory, which the host's engine does not see.
shared, outside := t.TempDir(), t.TempDir()
merged := filepath.Join(outside, "merged")
r, engine := checked(shared)
asItRan := buildForTheMerge(t.Context(), engine.run, spec, CheckBuilding{Publish: r, Scratch: newScratch(), Held: held},
outside, merged, judged.Merge, judged, GitCredential{}, nil, logging(nil))
if builtCount(asItRan) != 0 {
t.Fatalf("the engine of this test sees the holder's temporary directory: %+v", asItRan)
}
// The check as it builds now: every module its merge moves, kept under its key.
scratch := newScratch()
spec.BuildWith = &CheckBuilding{Publish: r, Scratch: scratch, Held: held}
built := forTheMerge(t.Context(), engine.run, spec, passed, shared, merged, judged, GitCredential{}, nil,
logging(nil))
if len(built) != 3 || builtCount(built) != 3 {
t.Fatalf("the check built %d of %d module(s) its merge moves: %+v", builtCount(built), len(built), built)
}
var names []NamedBuild
for _, b := range built {
n := NamedBuild{Pull: pull, Check: spec.ID, Module: b.Module, Key: b.Key, Digests: map[string]string{}}
for _, m := range b.Made {
d := m.Digest
if d == "" {
d = DigestOf(m.Reference)
}
n.Digests[m.Name] = d
}
names = append(names, n)
}
// The walk of the merge commit, tier by tier: each module taken from the check, none built.
walked := map[string]string{"mesh-controller": "https://forge.invalid/mesh-controller.git"}
for module, repository := range repositories {
walked[module] = repository
}
for _, module := range []string{"mesh-controller", "build-agent", "route-proxy"} {
w, walk := checked(shared)
w.secondary[walked["mesh-controller"]] = aGoModule("mesh-controller")
got, err := BuildReusing(t.Context(), walk.run, w, Reuse{Scratch: scratch, Pulls: []PullRef{pull}, Named: names},
walked[module], "", "main", t.TempDir(), held, Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatalf("%s: %v", module, err)
}
if got.Reused == nil || got.Why != "" {
t.Fatalf("the walk built %s again: %s", module, got.Why)
}
for _, line := range w.ran {
if strings.HasPrefix(line, "docker run") && strings.Contains(line, "go build") {
t.Fatalf("the walk compiled %s: %s", module, line)
}
}
}
}
+158
View File
@@ -0,0 +1,158 @@
package builder
import (
"context"
"errors"
"path/filepath"
"strings"
"sync"
"testing"
"time"
)
// novox/hq issue 716: a check built the images its merge moves one after another, and only after its merge-check.sh
// had passed — 3 to 6 minutes on a check of about 5. The modules of a tier stand on none of each other and are built
// at the same time; with a judge cache, beside the tests; a check that does not pass lets go of them.
// atOnce is a fake engine whose compiles wait until every one of a tier's has started: builds one after another never
// get there, and the first one's compile says so.
type atOnce struct {
mu sync.Mutex
r *recorded
want int
started int
all chan struct{}
shared string
}
func (a *atOnce) run(ctx context.Context, dir, name string, args ...string) (string, error) {
if name == "docker" && len(args) > 0 && args[0] == "run" && strings.Contains(strings.Join(args, " "), "go build") {
a.mu.Lock()
a.started++
if a.started == a.want {
close(a.all)
}
a.mu.Unlock()
select {
case <-a.all:
case <-time.After(5 * time.Second):
return "", errors.New("built alone: no other module of its tier was being built")
}
}
a.mu.Lock()
defer a.mu.Unlock()
return onTrunk{recorded: a.r}.run(ctx, dir, name, args...)
}
func (a *atOnce) PublishImage(ctx context.Context, localTag, repository string) (string, error) {
a.mu.Lock()
defer a.mu.Unlock()
return a.r.PublishImage(ctx, localTag, repository)
}
func (a *atOnce) PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error) {
a.mu.Lock()
defer a.mu.Unlock()
return a.r.PublishArchive(ctx, repository, body, digest)
}
// aLockedScratch is the scratch namespace in memory, said to by builds at once.
type aLockedScratch struct {
mu sync.Mutex
*aScratch
discarded []CheckBuild
}
func (s *aLockedScratch) Keep(ctx context.Context, p PullRef, c string, at time.Time, b []CheckBuild) error {
s.mu.Lock()
defer s.mu.Unlock()
return s.aScratch.Keep(ctx, p, c, at, b)
}
func (s *aLockedScratch) Find(ctx context.Context, module, key string) (CheckBuild, bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
return s.aScratch.Find(ctx, module, key)
}
func (s *aLockedScratch) Discard(_ context.Context, b []CheckBuild) error {
s.mu.Lock()
defer s.mu.Unlock()
s.discarded = append(s.discarded, b...)
return nil
}
// threeAtOnce are three Go modules in one tier, each from its own repository.
func threeAtOnce(t *testing.T) (*atOnce, CheckSpec, map[string]string) {
t.Helper()
r := &recorded{tree: "tree-of-the-merge", contents: aGoModule("mesh-controller"),
secondary: map[string]map[string]string{}}
spec := CheckSpec{ID: "build-716", Owner: "novox", Repo: "mesh-controller", Number: 716,
Builds: []CheckBuildSpec{{Module: "mesh-controller", Merged: true, Tier: 0}}}
for _, module := range []string{"build-agent", "route-proxy"} {
repository := "https://forge.invalid/" + module + ".git"
r.secondary[repository] = aGoModule(module)
spec.Builds = append(spec.Builds, CheckBuildSpec{Module: module, Repository: repository, Ref: "main", Tier: 0})
}
held := map[string]string{"mesh-tools-go/build": "registry.invalid/mesh-tools-go/build@sha256:" + strings.Repeat("b", 64)}
return &atOnce{r: r, want: 3, all: make(chan struct{})}, spec, held
}
// The modules of one tier are built at the same time, each in a working copy of its own.
func TestReplay716TheModulesOfATierAreBuiltAtTheSameTime(t *testing.T) {
withVersion(t, "builder-1")
engine, spec, held := threeAtOnce(t)
scratch := &aLockedScratch{aScratch: newScratch()}
built := buildForTheMerge(t.Context(), engine.run, spec, CheckBuilding{Publish: engine, Scratch: scratch, Held: held},
t.TempDir(), filepath.Join(t.TempDir(), "merged"), "2d22279b1d07",
CheckVerdict{Head: "98e856874913", Trunk: "c975dbf7b053"}, GitCredential{}, nil, logging(nil))
if builtCount(built) != 3 {
t.Fatalf("the check built %d of 3 module(s) of one tier: %+v", builtCount(built), built)
}
if len(scratch.builds) != 3 {
t.Fatalf("the check kept %d build(s), not 3", len(scratch.builds))
}
}
// With a judge cache, the builds run while the tests run, where no container of the check reaches; a check that passed
// keeps them, and one that failed lets go of every one and names none.
func TestReplay716TheBuildsRunBesideTheTestsAndAFailedCheckLetsGoOfThem(t *testing.T) {
withVersion(t, "builder-1")
workspace := t.TempDir()
if _, err := buildsBesideTheTests(filepath.Join(workspace, "judge-cache"), workspace, "build-716"); err == nil {
t.Fatal("a judge cache inside the workspace was built in, where a pull request's script reaches")
}
for _, passed := range []bool{true, false} {
cache := t.TempDir()
place, err := buildsBesideTheTests(cache, workspace, "build-716")
if err != nil || !withinDir(cache, place) || withinDir(workspace, place) {
t.Fatalf("the builds' place %q (%v) is not the judge cache's, outside the workspace", place, err)
}
engine, spec, held := threeAtOnce(t)
scratch := &aLockedScratch{aScratch: newScratch()}
spec.BuildWith = &CheckBuilding{Publish: engine, Scratch: scratch, Held: held}
judged := CheckVerdict{Head: "98e856874913", Trunk: "c975dbf7b053", Merge: "2d22279b1d07"}
beside := startBesideTheTests(t.Context(), engine.run, spec, place, filepath.Join(t.TempDir(), "merged"),
judged, GitCredential{}, nil, logging(nil))
// The tests: the builds are under way before the check has a verdict.
select {
case <-engine.all:
case <-time.After(5 * time.Second):
t.Fatal("nothing was built while the tests ran")
}
verdict := CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "pass"}}
if !passed {
verdict.Repo.Verdict = "fail"
}
built := beside.settle(t.Context(), verdict)
beside.abandon()
switch {
case passed && (builtCount(built) != 3 || len(scratch.builds) != 3 || len(scratch.discarded) != 0):
t.Fatalf("a check that passed: built %+v, kept %d, let go of %d", built, len(scratch.builds),
len(scratch.discarded))
case !passed && (built != nil || len(scratch.builds) != 0 || len(scratch.discarded) != 3):
t.Fatalf("a check that failed: named %+v, kept %d, let go of %d", built, len(scratch.builds),
len(scratch.discarded))
}
}
}
+12 -4
View File
@@ -494,11 +494,15 @@ func (r Registry) Discard(ctx context.Context, builds []CheckBuild) error {
func (r Registry) letGo(ctx context.Context, checks, all map[string]checkIndex) (int, error) {
store := r.store()
stays := map[string]bool{}
// recorded are the records under a key another check still names (novox/hq issue 703): a later check that kept
// the build names it as the check's that built it, and the walk finds it by that record.
recorded := map[string]bool{}
for tag, c := range all {
if _, going := checks[tag]; going {
continue
}
for _, b := range c.Builds {
recorded[b.Module+" "+b.Key+" "+b.Check] = true
for _, m := range b.Made {
stays[catalogue.Recorded(m.Reference)] = true
}
@@ -523,8 +527,10 @@ func (r Registry) letGo(ctx context.Context, checks, all map[string]checkIndex)
}
}
// The record under the key, only while it is still this check's: a later check of the same key
// replaced it, and that one is its own check's to let go of.
if kept, found, err := r.Find(ctx, b.Module, b.Key); err == nil && found && kept.Check == c.Check {
// replaced it, and that one is its own check's to let go of. A build a check kept is that build's
// check's record, and goes with the last check naming it (novox/hq issue 703).
if kept, found, err := r.Find(ctx, b.Module, b.Key); err == nil && found &&
(kept.Check == c.Check || kept.Check == b.Check) && !recorded[b.Module+" "+b.Key+" "+kept.Check] {
if err := store.DeleteTagged(ctx, ScratchPrefix+b.Module, tagOf(b.Key)); err != nil && !errors.Is(err, artifacts.Gone) {
ok = false
failed = append(failed, fmt.Sprintf("%s's record: %v", b.Module, err))
@@ -610,6 +616,8 @@ func unnamed(named []NamedBuild, module, key string, found CheckBuild) string {
return ""
}
}
return "a build is kept under this key, and no check verdict of the walk's pull requests names it with its " +
"digests: not taken"
// A build of another pull request, or of a head the forge did not merge, is in no verdict the walk is handed
// (novox/hq issue 703): said with the check that made it, whose head names which head it was.
return "a build is kept under this key, by " + found.Said() + ", and no check verdict of the walk's pull requests " +
"at the head the forge merged names it with its digests: not taken"
}
+153 -1
View File
@@ -6,6 +6,7 @@ import (
"errors"
"net/http"
"net/http/httptest"
"reflect"
"slices"
"strconv"
"strings"
@@ -227,7 +228,8 @@ func TestARecordNoVerdictNamesIsRefusedAndBuilt(t *testing.T) {
if got.Reused != nil || len(scratch.took) > 0 || !dockerBuilt(r) {
t.Fatalf("a record no verdict names was taken: %+v, copied %v", got.Reused, scratch.took)
}
if !strings.Contains(got.Why, "no check verdict of the walk's pull requests names it") {
if !strings.Contains(got.Why, "no check verdict of the walk's pull requests at the head the forge merged names it") ||
!strings.Contains(got.Why, "at head abcdefabcdef") {
t.Fatalf("the walk says %q", got.Why)
}
})
@@ -637,3 +639,153 @@ func TestABuildTwoChecksShareGoesWithTheLastOfThem(t *testing.T) {
t.Fatal("the build stayed after the last check naming it went")
}
}
// aCheckAgain is a later check (build-2) of aPull building the module from a tree, with the builds the build seat's
// verdicts name for it (novox/hq issue 703).
func aCheckAgain(t *testing.T, scratch *aScratch, tree string, leave []NamedBuild) (CheckBuilt, *recorded) {
t.Helper()
return aCheckOf(t, scratch, aPull, "abcdefabcdef", tree, leave)
}
// aCheckOf is a later check (build-2) of a pull request at a head, the trunk moved since build-1.
func aCheckOf(t *testing.T, scratch *aScratch, pull PullRef, head, tree string, leave []NamedBuild) (CheckBuilt, *recorded) {
t.Helper()
r, run, workspace := aModuleAt(t, tree)
spec := CheckSpec{ID: "build-2", Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number,
Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod",
Ref: "main"}}}
built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: scratch, Named: leave},
workspace, "", "", CheckVerdict{Head: head, Trunk: "654321654321"}, GitCredential{}, nil, logging(nil))
if len(built) != 1 || built[0].NotBuilt != "" || built[0].Key == "" {
t.Fatalf("the second check did not answer the module for its merge: %+v", built)
}
return built[0], r
}
// A check asked again when only the trunk moved, whose key is unchanged, builds nothing: it keeps the build an
// earlier verdict names and names it by key and digest, so the walk can take it (novox/hq issue 703).
func TestASecondCheckWhoseKeyIsUnchangedBuildsNothingAndNamesTheEarlierBuild(t *testing.T) {
withVersion(t, "builder-1")
scratch := newScratch()
first := aCheckBuilt(t, scratch, "tree-one")
again, r := aCheckAgain(t, scratch, "tree-one", named(first))
if dockerBuilt(r) || len(r.images) > 0 || len(r.archives) > 0 {
t.Fatalf("the second check built again: ran %v, published %v %v", r.ran, r.images, r.archives)
}
if again.Key != first.Key || again.Kept != "build-1" || !reflect.DeepEqual(again.Made, first.Made) {
t.Fatalf("the second check names %+v, not the first check's build %+v", again, first)
}
// The walk takes it by what the second verdict names.
leave := named(again)
leave[0].Check = again.Kept
w, run, workspace := aModuleAt(t, "tree-one")
got, err := BuildReusing(t.Context(), run, w, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: leave},
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
if err != nil || got.Reused == nil || dockerBuilt(w) {
t.Fatalf("the walk did not take the kept build: %v, why %q", err, got.Why)
}
}
// A check whose key changed builds, as before.
func TestASecondCheckWhoseKeyChangedBuilds(t *testing.T) {
withVersion(t, "builder-1")
scratch := newScratch()
first := aCheckBuilt(t, scratch, "tree-one")
again, r := aCheckAgain(t, scratch, "tree-two", named(first))
if !dockerBuilt(r) || again.Kept != "" || again.Key == first.Key {
t.Fatalf("a check whose key changed did not build: %+v", again)
}
}
// A scratch record no verdict names is never kept: the store takes writes from anything on its network, so a
// check builds rather than name a planted record (novox/hq issue 703, ADR 0350 §2).
func TestASecondCheckKeepsNoRecordAVerdictDoesNotName(t *testing.T) {
withVersion(t, "builder-1")
scratch := newScratch()
first := aCheckBuilt(t, scratch, "tree-one")
for name, leave := range map[string][]NamedBuild{
"no verdict": nil,
"another digest": func() []NamedBuild {
n := named(first)
n[0].Digests["server"] = "sha256:" + strings.Repeat("e", 64)
return n
}(),
"another key": func() []NamedBuild { n := named(first); n[0].Key = "r1:" + strings.Repeat("f", 64); return n }(),
} {
t.Run(name, func(t *testing.T) {
again, r := aCheckAgain(t, scratch, "tree-one", leave)
if !dockerBuilt(r) || again.Kept != "" {
t.Fatalf("a record no verdict names was kept: %+v", again)
}
})
}
}
// A build a later check kept stays when the check that built it goes: its artifacts and its record, which the later
// check's verdict names (novox/hq issue 703, ADR 0350 §5).
func TestABuildALaterCheckKeptStaysWhenTheCheckThatBuiltItGoes(t *testing.T) {
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
server := httptest.NewServer(m.serve())
defer server.Close()
address := strings.TrimPrefix(server.URL, "http://")
registry := Registry{Address: address}
ctx := t.Context()
now := time.Date(2026, 10, 14, 9, 0, 0, 0, time.UTC)
body := `{"mediaType":"` + mediaManifestOCI + `","layers":[]}`
digest := digestOf([]byte(body))
m.manifests["scratch/meshboard/server@"+digest] = []byte(body)
key := "r1:" + strings.Repeat("1", 64)
b := CheckBuild{Key: key, Module: "meshboard", Pull: aPull, Check: "build-1", At: now,
Made: []catalogue.Built{{Name: "server", Kind: "image", Reference: address + "/scratch/meshboard/server@" + digest}}}
if err := registry.Keep(ctx, aPull, "build-1", now, []CheckBuild{b}); err != nil {
t.Fatal(err)
}
// Another pull request's check keeps it: its own document names the build, which stays the first check's.
other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8}
if err := registry.Keep(ctx, other, "build-2", now, []CheckBuild{b}); err != nil {
t.Fatal(err)
}
held := func() bool { _, ok := m.manifests["scratch/meshboard/server@"+digest]; return ok }
found := func() bool { _, ok, _ := registry.Find(ctx, "meshboard", key); return ok }
if n, err := registry.Forget(ctx, aPull); err != nil || n != 1 {
t.Fatalf("forgot %d (%v)", n, err)
}
if !held() || !found() {
t.Fatalf("the build a later check names went with the check that built it: held %v, record %v", held(), found())
}
if n, err := registry.Forget(ctx, other); err != nil || n != 1 {
t.Fatalf("forgot %d (%v)", n, err)
}
if held() || found() {
t.Fatalf("the build stayed after the last check naming it went: held %v, record %v", held(), found())
}
}
// Only an earlier check of the same pull request at the same head is kept from: another head of it, or another pull
// request whose verdict names the same key and digests, builds (novox/hq issue 703).
func TestACheckKeepsNoBuildOfAnotherHeadOrPullRequest(t *testing.T) {
withVersion(t, "builder-1")
other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8}
for name, c := range map[string]struct {
pull PullRef
head string
}{
"another head": {aPull, "999999999999"},
"another pull request": {other, "abcdefabcdef"},
} {
t.Run(name, func(t *testing.T) {
scratch := newScratch()
first := aCheckBuilt(t, scratch, "tree-one")
leave := named(first)
leave[0].Pull = c.pull // as if its own verdict named it
again, r := aCheckOf(t, scratch, c.pull, c.head, "tree-one", leave)
if !dockerBuilt(r) || again.Kept != "" {
t.Fatalf("a build of %s was kept: %+v", name, again)
}
})
}
}
+180
View File
@@ -0,0 +1,180 @@
package builder
import (
"archive/tar"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/catalogue"
)
// A module moves only when its program changes (novox/hq issue 721). A merge check builds every module the delivery
// plan moves; a module whose every artifact comes out the same as the build the mesh holds of it — the same files,
// the same modes and owners, the same entrypoint and environment — is said unchanged, and the walk leaves it where it
// is. **The whole image is compared, never its main binary alone**: a module whose image differs in any file moves.
// What is ignored is only what a rebuild cannot help changing: each file's time, the image's labels and its creation
// time. **A comparison that cannot be made says nothing**, and the module is built and moved as before.
// sameAsHeld says whether every artifact made is the same as the build the mesh holds of the module: the sentence
// that says so and, per artifact, the held reference it was compared with. Empty when any artifact differs, is not
// an image, has no held build, or could not be compared.
func sameAsHeld(ctx context.Context, run Runner, dir, module string, made []catalogue.Built,
held map[string]string) (string, map[string]string, error) {
if len(made) == 0 {
return "", nil, nil
}
against := map[string]string{}
for _, m := range made {
if m.Kind != "image" || m.Reference == "" {
return "", nil, nil
}
was := held[module+"/"+m.Name]
if was == "" {
return "", nil, nil
}
if digestPart(was) != "" && digestPart(was) == digestPart(m.Reference) {
against[m.Name] = was
continue
}
now, err := imageContent(ctx, run, dir, m.Reference)
if err != nil {
return "", nil, fmt.Errorf("%s's new image: %w", m.Name, err)
}
before, err := imageContent(ctx, run, dir, was)
if err != nil {
return "", nil, fmt.Errorf("%s's held image: %w", m.Name, err)
}
if now != before {
return "", nil, nil
}
against[m.Name] = was
}
var names []string
for name, ref := range against {
names = append(names, name+" "+shortRef(ref))
}
sort.Strings(names)
return "unchanged: its program is the same as running build " + strings.Join(names, ", "), against, nil
}
// digestPart is a reference's "sha256:…", empty when it is not pinned.
func digestPart(ref string) string {
if i := strings.LastIndex(ref, "@"); i >= 0 {
return ref[i+1:]
}
return ""
}
// shortRef names a reference by its digest's first twelve characters when it has one.
func shortRef(ref string) string {
if d := strings.TrimPrefix(digestPart(ref), "sha256:"); d != "" {
return twelve(d)
}
return ref
}
// imageContent is an image's content as one digest: its platform, its configuration without labels, and every file
// of its filesystem by path, type, mode, owner, link, extended attributes and bytes — never by time.
func imageContent(ctx context.Context, run Runner, dir, ref string) (string, error) {
if _, err := run(ctx, dir, "docker", "pull", "--quiet", ref); err != nil {
return "", fmt.Errorf("pulling %s: %w", ref, err)
}
config, err := run(ctx, dir, "docker", "image", "inspect", "--format",
"{{.Os}}/{{.Architecture}}/{{.Variant}} {{json .Config}}", ref)
if err != nil {
return "", fmt.Errorf("reading %s's configuration: %w", ref, err)
}
said, err := configContent(config)
if err != nil {
return "", fmt.Errorf("reading %s's configuration: %w", ref, err)
}
id, err := run(ctx, dir, "docker", "create", "--entrypoint", "/", ref)
if err != nil {
return "", fmt.Errorf("opening %s's filesystem: %w", ref, err)
}
id = strings.TrimSpace(id)
if id == "" {
return "", errors.New("the container engine named no container for " + ref)
}
defer func() { _, _ = run(context.WithoutCancel(ctx), dir, "docker", "rm", "--force", id) }()
if err := os.MkdirAll(dir, 0o700); err != nil {
return "", err
}
file := filepath.Join(dir, "content-"+twelve(id)+".tar")
defer os.Remove(file)
if _, err := run(ctx, dir, "docker", "export", "--output", file, id); err != nil {
return "", fmt.Errorf("reading %s's filesystem: %w", ref, err)
}
f, err := os.Open(file)
if err != nil {
return "", err
}
defer f.Close()
files, err := treeContent(f)
if err != nil {
return "", fmt.Errorf("reading %s's filesystem: %w", ref, err)
}
sum := sha256.Sum256([]byte(said + "\n" + files))
return "sha256:" + hex.EncodeToString(sum[:]), nil
}
// configContent is an image's platform and configuration as compared: Labels, the parent's id and the hostname the
// builder ran under are left out — the first holds the commit, the others no part of what runs.
func configContent(inspected string) (string, error) {
platform, raw, ok := strings.Cut(strings.TrimSpace(inspected), " ")
if !ok {
return "", fmt.Errorf("unexpected inspection %q", inspected)
}
var config map[string]json.RawMessage
if err := json.Unmarshal([]byte(raw), &config); err != nil {
return "", err
}
for _, k := range []string{"Labels", "Image", "Hostname", "Domainname"} {
delete(config, k)
}
out, err := json.Marshal(config) // map keys are sorted
if err != nil {
return "", err
}
return platform + " " + string(out), nil
}
// treeContent is a filesystem tarball's entries, sorted by path, each with everything but its times.
func treeContent(r io.Reader) (string, error) {
tr := tar.NewReader(r)
var lines []string
for {
h, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return "", err
}
sum := sha256.New()
if _, err := io.Copy(sum, tr); err != nil {
return "", err
}
var xattrs []string
for k, v := range h.PAXRecords {
if strings.HasPrefix(k, "SCHILY.xattr.") {
xattrs = append(xattrs, k+"="+hex.EncodeToString([]byte(v)))
}
}
sort.Strings(xattrs)
lines = append(lines, fmt.Sprintf("%q %c %o %d:%d %q %d,%d %d %x %s",
strings.TrimPrefix(h.Name, "./"), h.Typeflag, h.Mode, h.Uid, h.Gid, h.Linkname, h.Devmajor, h.Devminor,
h.Size, sum.Sum(nil), strings.Join(xattrs, ";")))
}
sort.Strings(lines)
return strings.Join(lines, "\n"), nil
}
+126
View File
@@ -0,0 +1,126 @@
package builder
import (
"archive/tar"
"bytes"
"context"
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
)
// aFilesystem is a filesystem tarball of files, each written at the time given.
func aFilesystem(t *testing.T, at time.Time, files map[string]string) []byte {
t.Helper()
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
for _, name := range []string{"usr/local/bin/mesh-builder", "etc/apk/world"} {
body, ok := files[name]
if !ok {
continue
}
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o755, Size: int64(len(body)), ModTime: at,
Typeflag: tar.TypeReg}); err != nil {
t.Fatal(err)
}
if _, err := tw.Write([]byte(body)); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
// anEngine answers the container engine's commands for images by reference: each image's filesystem and labels.
func anEngine(images map[string][]byte, labels map[string]string, failing string) Runner {
created := map[string]string{}
return func(_ context.Context, _ string, name string, args ...string) (string, error) {
joined := strings.Join(args, " ")
ref := args[len(args)-1]
if failing != "" && strings.Contains(joined, failing) {
return "", fmt.Errorf("the engine refused %s", joined)
}
switch {
case args[0] == "pull", args[0] == "rm":
return "", nil
case args[0] == "image":
return fmt.Sprintf(`linux/amd64/ {"Entrypoint":["/usr/local/bin/mesh-builder"],"Labels":{"commit":%q}}`,
labels[ref]), nil
case args[0] == "create":
id := fmt.Sprintf("container%04d", len(created))
created[id] = ref
return id + "\n", nil
case args[0] == "export":
return "", os.WriteFile(args[2], images[created[ref]], 0o600)
}
return "", fmt.Errorf("unexpected %s %s", name, joined)
}
}
// TestAModuleWhoseProgramDidNotChangeIsSaidUnchanged is novox/hq issue 721: mesh-controller #335 changed one sentence
// in a package mesh-builder imports and never reaches, and build-agent moved. Built again, its image differs only in
// the files' times and its commit label, and the check says it is the same as the build the mesh holds.
func TestAModuleWhoseProgramDidNotChangeIsSaidUnchanged(t *testing.T) {
held := map[string]string{"build-agent/agent": "store/build-agent@sha256:aaaa"}
made := []catalogue.Built{{Name: "agent", Kind: "image", Reference: "scratch/build-agent@sha256:bbbb"}}
files := map[string]string{"usr/local/bin/mesh-builder": "the same program", "etc/apk/world": "docker-cli\ngit\n"}
images := map[string][]byte{
"store/build-agent@sha256:aaaa": aFilesystem(t, time.Unix(1000, 0), files),
"scratch/build-agent@sha256:bbbb": aFilesystem(t, time.Unix(2000, 0), files),
}
labels := map[string]string{"store/build-agent@sha256:aaaa": "a5500f73", "scratch/build-agent@sha256:bbbb": "c2ec0a60"}
same, against, err := sameAsHeld(context.Background(), anEngine(images, labels, ""), t.TempDir(), "build-agent",
made, held)
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(same, "unchanged: its program is the same as running build agent") ||
against["agent"] != held["build-agent/agent"] {
t.Fatalf("an image the same but for its times and labels is not said unchanged: %q %v", same, against)
}
// **Any file differing moves it**, not only its program.
changed := map[string]string{"usr/local/bin/mesh-builder": "the same program", "etc/apk/world": "docker-cli\ngit\ncurl\n"}
images["scratch/build-agent@sha256:bbbb"] = aFilesystem(t, time.Unix(2000, 0), changed)
if same, _, err := sameAsHeld(context.Background(), anEngine(images, labels, ""), t.TempDir(), "build-agent", made,
held); err != nil || same != "" {
t.Fatalf("an image with another file is said unchanged: %q %v", same, err)
}
// **A comparison that fails says nothing**, and the module moves.
images["scratch/build-agent@sha256:bbbb"] = aFilesystem(t, time.Unix(2000, 0), files)
if same, _, err := sameAsHeld(context.Background(), anEngine(images, labels, "export"), t.TempDir(), "build-agent",
made, held); err == nil || same != "" {
t.Fatalf("a comparison that failed said %q, %v", same, err)
}
// No held build, or an artifact that is not an image: nothing compared.
if same, _, _ := sameAsHeld(context.Background(), anEngine(images, labels, ""), t.TempDir(), "build-agent", made,
nil); same != "" {
t.Fatalf("a module the mesh holds no build of is said unchanged: %q", same)
}
archive := []catalogue.Built{{Name: "agent", Kind: "archive", Reference: "scratch/build-agent@sha256:bbbb"}}
if same, _, _ := sameAsHeld(context.Background(), anEngine(images, labels, ""), t.TempDir(), "build-agent",
archive, held); same != "" {
t.Fatalf("an archive is said unchanged without being compared: %q", same)
}
}
// TestAnImageWithAnotherEntrypointIsNotTheSame: the configuration counts, but for its labels.
func TestAnImageWithAnotherEntrypointIsNotTheSame(t *testing.T) {
a, err := configContent(`linux/amd64/ {"Entrypoint":["/a"],"Labels":{"commit":"1"}}`)
if err != nil {
t.Fatal(err)
}
b, _ := configContent(`linux/amd64/ {"Labels":{"commit":"2"},"Entrypoint":["/a"]}`)
c, _ := configContent(`linux/amd64/ {"Entrypoint":["/b"],"Labels":{"commit":"1"}}`)
if a != b || a == c {
t.Fatalf("labels count, or the entrypoint does not: %q %q %q", a, b, c)
}
}
+12 -11
View File
@@ -14,14 +14,15 @@ import (
"github.com/novox/mesh-controller/internal/catalogue"
)
// withTemplateTexts is what was built with each template's text, for a module whose files name templates
// (novox/hq ADR 0349 §2): its templates archive gains the text of every file in it, read back from the packed
// bytes, by its path inside the archive. Resolve reads them into the files' content.
// withTemplateTexts is what was built with each template's text, for a module whose files name templates (novox/hq ADR
// 0349 §2): its templates archive — the one built from its templates directory, files/ unless its manifest names
// another (issue 731) — gains the text of every file in it, read back from the packed bytes, by its path inside the
// archive. Resolve reads them into the files' content.
//
// **Read from the archive, proven by its digest.** The module's files/ is packed again, as the build packs it,
// and the bytes are read only when they hash to the digest the build made or took: a fresh build's own archive,
// or a check's build a walk copied by digest without building (novox/hq ADR 0350 §2). Packing is deterministic,
// so one source packs to one digest; a tree that packs to another is not the archive, and its text is refused.
// **Read from the archive, proven by its digest.** The module's templates directory is packed again, as the build packs
// it, and the bytes are read only when they hash to the digest the build made or took: a fresh build's own archive, or
// a check's build a walk copied by digest without building (novox/hq ADR 0350 §2). Packing is deterministic, so one
// source packs to one digest; a tree that packs to another is not the archive, and its text is refused.
func withTemplateTexts(manifest catalogue.Manifest, within string, built []catalogue.Built) ([]catalogue.Built, error) {
if !manifest.Templated() {
return built, nil
@@ -35,18 +36,18 @@ func withTemplateTexts(manifest catalogue.Manifest, within string, built []catal
}
if name == "" || at < 0 {
return nil, fmt.Errorf("%s names templates and built no archive of %s/ to read them from",
manifest.Module, catalogue.TemplatesDir)
manifest.Module, manifest.TemplatesDir())
}
body, err := pack(filepath.Join(within, catalogue.TemplatesDir))
body, err := pack(filepath.Join(within, filepath.FromSlash(manifest.TemplatesDir())))
if err != nil {
return nil, fmt.Errorf("%s: packing %s/ to read its templates back failed: %w", manifest.Module,
catalogue.TemplatesDir, err)
manifest.TemplatesDir(), err)
}
sum := sha256.Sum256(body)
digest := "sha256:" + hex.EncodeToString(sum[:])
if built[at].Digest != digest {
return nil, fmt.Errorf("%s: %s/ packs to the digest %s, and the archive %q built or taken is %s: the "+
"templates are read only from the archive the mesh records", manifest.Module, catalogue.TemplatesDir,
"templates are read only from the archive the mesh records", manifest.Module, manifest.TemplatesDir(),
digest, name, orNone(built[at].Digest))
}
texts, err := unpackTexts(body)
+45
View File
@@ -2,6 +2,8 @@ package builder
import (
"context"
"crypto/sha256"
"encoding/hex"
"os"
"path/filepath"
"strings"
@@ -134,3 +136,46 @@ func writeTree(t *testing.T, dir string, files map[string]string) string {
}
return dir
}
// A module whose files/ is an archive unpacked onto the machine keeps its templates in a directory it names (issue
// 731): the build reads each template back from the archive packed from that directory, and files/ is packed as it
// was, its templates nowhere in it.
const unpackingModule = `{"module":"meshboard","version":"1",
"templates":"templates",
"build":{"artifacts":[
{"name":"scripts","kind":"archive","from":"files"},
{"name":"templates","kind":"archive","from":"templates"}]},
"resources":[
{"id":"scripts","type":"archive","path":"/usr/local/lib/meshboard","artifact":"scripts"},
{"id":"conf","type":"file","path":"/etc/meshboard/meshboard.conf","template":"templates/meshboard.conf"}]}`
func TestABuildReadsEachTemplateBackFromTheTemplatesDirectoryItsManifestNames(t *testing.T) {
files := map[string]string{"files/bin/meshboard-hook": "#!/bin/sh\n", "templates/meshboard.conf": "listen = 8080\n"}
r, workspace := aRepository(t, unpackingModule, files)
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil,
Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if text := contentOf(t, got.Manifest, "conf"); text != files["templates/meshboard.conf"] {
t.Fatalf("the template's text is %q, not the file's", text)
}
if got.Manifest.Templates != "" {
t.Fatalf("the built manifest still names its templates directory: %q", got.Manifest.Templates)
}
// files/ packs exactly as it would with no templates beside it.
alone, err := pack(writeTree(t, t.TempDir(), map[string]string{"bin/meshboard-hook": "#!/bin/sh\n"}))
if err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(alone)
packed := ""
for _, b := range got.Built {
if b.Name == "scripts" {
packed = b.Digest
}
}
if packed != "sha256:"+hex.EncodeToString(sum[:]) {
t.Fatalf("files/ packed to %q, not to what it packs to alone", packed)
}
}
+80
View File
@@ -0,0 +1,80 @@
package builder
import (
"os"
"os/exec"
"path/filepath"
"sync"
"testing"
)
// **A judge's build in a test starts from a warm Go cache** (novox/hq issue 717). The runtime these tests put on
// PATH (aRuntimeThatBuildsTheJudge) runs the check's `go build` and `go test` on this machine with the Go cache the
// check names — a fresh directory in every test — so each of them compiled the standard library from nothing: some
// ten CPU-seconds a build, under twenty tests, half of the CPU the whole suite spent. The runtime now fills an empty
// Go cache it is given from one this process warmed once, before it runs `go`; what the check names, mounts and
// keeps is unchanged, and so is what each test reads of it. A cache entry is keyed by everything the compile
// depends on, flags included, so a build with a planted -toolexec hook still misses it and still runs its hook.
//
// The warm cache lives beside the system's temporary directory as it was when the package started, never under a
// TMPDIR a test set and then asserts left empty, and is removed when the package's tests end.
var (
warmCacheBase = os.TempDir()
warmCacheOnce sync.Once
warmCacheDir string
)
func TestMain(m *testing.M) {
code := m.Run()
if warmCacheDir != "" {
_ = os.RemoveAll(warmCacheDir)
}
os.Exit(code)
}
// warmGoCache is a Go cache holding what a judge's build and the replays' `go test` compile of the standard
// library, or "" when it cannot be warmed — a test then builds cold, as before, and is only slower.
func warmGoCache(t *testing.T) string {
t.Helper()
warmCacheOnce.Do(func() {
dir, err := os.MkdirTemp(warmCacheBase, "judge-warm-go-cache-")
if err != nil {
return
}
src := filepath.Join(dir, "src")
files := map[string]string{
"go.mod": "module example.org/warm\n\ngo 1.22\n",
"main.go": "package main\n\nimport (\n\t\"fmt\"\n\t\"os\"\n)\n\nfunc main() { fmt.Fprintln(os.Stdout, len(os.Args)) }\n",
"main_test.go": "package main\n\nimport \"testing\"\n\nfunc TestWarm(t *testing.T) {}\n",
}
for name, body := range files {
if err := os.MkdirAll(src, 0o755); err != nil {
return
}
if err := os.WriteFile(filepath.Join(src, name), []byte(body), 0o644); err != nil {
return
}
}
cache := filepath.Join(dir, "go-cache")
env := append(os.Environ(), "GOCACHE="+cache, "GOENV=off", "GOTOOLCHAIN=local", "GOPROXY=off", "GOWORK=off",
"GOFLAGS=-buildvcs=false", "TMPDIR="+warmCacheBase)
// As the check builds a judge (CGO off) and runs the replays (CGO as the toolchain has it).
for _, run := range [][]string{
{"env", "CGO_ENABLED=0", "go", "build", "-o", filepath.Join(dir, "judge"), "."},
{"go", "test", "-count=1", "."},
} {
cmd := exec.Command(run[0], run[1:]...)
cmd.Dir, cmd.Env = src, env
if out, err := cmd.CombinedOutput(); err != nil {
t.Logf("the warm Go cache could not be built, so judges build cold: %v\n%s", err, out)
_ = os.RemoveAll(dir)
return
}
}
warmCacheDir = dir
})
if warmCacheDir == "" {
return ""
}
return filepath.Join(warmCacheDir, "go-cache")
}