jschoubben
060f2a9ca5
mesh/merge-gate head 060f2a9ca5fd merged into trunk 04730b54a8f2 — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head 060f2a9ca5fd merged into trunk 04730b54a8f2 — pass: its merge-check.sh passed
mesh/delivery superseded: a newer head of the same pull request
The gate and the repository's check mount the workspace read-write, so a build from that tree could hold what the tests wrote under a key that describes the merge. A check now clones the merge outside the workspace before anything of the change runs, and builds from that. A check keeps, and a walk takes, only a build an earlier check of the same pull request at the same head made: another pull request's build, or an earlier head's, is never registered. The walk reads the verdicts once (novox/hq issue 703, review of #328).
779 lines
32 KiB
Go
779 lines
32 KiB
Go
package builder
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"reflect"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
)
|
|
|
|
// One build per change (novox/hq ADR 0350): the merge check builds what the merge would move into the scratch
|
|
// namespace under its reuse key, and the walk of the trunk commit takes that build by digest instead of building.
|
|
|
|
// aScratch is the scratch namespace in memory: check builds by module and key, and what the walk took.
|
|
type aScratch struct {
|
|
builds map[string]CheckBuild
|
|
took []string
|
|
failTake bool
|
|
}
|
|
|
|
func newScratch() *aScratch { return &aScratch{builds: map[string]CheckBuild{}} }
|
|
|
|
func (s *aScratch) Keep(_ context.Context, _ PullRef, _ string, _ time.Time, builds []CheckBuild) error {
|
|
for _, b := range builds {
|
|
s.builds[b.Module+" "+b.Key] = b
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *aScratch) Find(_ context.Context, module, key string) (CheckBuild, bool, error) {
|
|
b, ok := s.builds[module+" "+key]
|
|
return b, ok, nil
|
|
}
|
|
|
|
func (s *aScratch) OfPulls(_ context.Context, pulls []PullRef) ([]CheckBuild, error) {
|
|
var out []CheckBuild
|
|
for _, b := range s.builds {
|
|
if slices.Contains(pulls, b.Pull) {
|
|
out = append(out, b)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (s *aScratch) Take(_ context.Context, module string, made catalogue.Built) (catalogue.Built, error) {
|
|
if s.failTake {
|
|
return catalogue.Built{}, errors.New("the store refused the mount")
|
|
}
|
|
s.took = append(s.took, made.Reference)
|
|
made.Reference = strings.Replace(made.Reference, ScratchPrefix, "", 1)
|
|
return made, nil
|
|
}
|
|
|
|
func (s *aScratch) Forget(context.Context, PullRef) (int, error) { return 0, nil }
|
|
func (s *aScratch) Expire(context.Context, time.Duration, time.Time) (int, error) { return 0, nil }
|
|
func (s *aScratch) Discard(context.Context, []CheckBuild) error { return nil }
|
|
|
|
const reusedModule = `{"module":"meshboard","version":"1",
|
|
"build":{"artifacts":[
|
|
{"name":"server","kind":"image","from":"Dockerfile"},
|
|
{"name":"look","kind":"archive","from":"files"}]},
|
|
"resources":[
|
|
{"id":"svc","type":"container","name":"meshboard","artifact":"server"},
|
|
{"id":"theme","type":"archive","path":"/opt/meshboard","artifact":"look"}]}`
|
|
|
|
// aModuleAt is a repository holding the module under mod/, whose directory git names as tree, on its trunk.
|
|
func aModuleAt(t *testing.T, tree string) (*recorded, Runner, string) {
|
|
t.Helper()
|
|
r := &recorded{tree: tree, contents: map[string]string{"mod/" + ManifestName: reusedModule,
|
|
"mod/Dockerfile": "FROM scratch", "mod/files/theme.conf": "dark"}}
|
|
run := func(ctx context.Context, dir, name string, args ...string) (string, error) {
|
|
if name == "git" && len(args) > 0 && args[0] == "symbolic-ref" {
|
|
r.ran = append(r.ran, name+" "+strings.Join(args, " "))
|
|
return "origin/main\n", nil
|
|
}
|
|
return r.run(ctx, dir, name, args...)
|
|
}
|
|
return r, run, t.TempDir()
|
|
}
|
|
|
|
func dockerBuilt(r *recorded) bool {
|
|
for _, line := range r.ran {
|
|
if strings.HasPrefix(line, "docker build") {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
var aPull = PullRef{Owner: "novox", Repo: "meshboard", Number: 7}
|
|
|
|
// aCheckBuilt is the check of aPull building the module from a tree, into the scratch namespace.
|
|
func aCheckBuilt(t *testing.T, scratch *aScratch, tree string) CheckBuilt {
|
|
t.Helper()
|
|
r, run, workspace := aModuleAt(t, tree)
|
|
spec := CheckSpec{ID: "build-1", Owner: aPull.Owner, Repo: aPull.Repo, Number: aPull.Number,
|
|
Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod",
|
|
Ref: "main"}}}
|
|
built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: scratch}, workspace, "", "",
|
|
CheckVerdict{Head: "abcdefabcdef", Trunk: "123456123456"}, GitCredential{}, nil, logging(nil))
|
|
if len(built) != 1 || built[0].NotBuilt != "" || built[0].Key == "" {
|
|
t.Fatalf("the check did not build the module for its merge: %+v", built)
|
|
}
|
|
if !dockerBuilt(r) {
|
|
t.Fatal("the check built no image")
|
|
}
|
|
for repository := range r.images {
|
|
if !strings.HasPrefix(repository, ScratchPrefix) {
|
|
t.Fatalf("the check published %s outside the scratch namespace", repository)
|
|
}
|
|
}
|
|
for repository := range r.archives {
|
|
if !strings.HasPrefix(repository, ScratchPrefix) {
|
|
t.Fatalf("the check published %s outside the scratch namespace", repository)
|
|
}
|
|
}
|
|
return built[0]
|
|
}
|
|
|
|
// named is the walk's leave naming what a check's verdict says it built, as the controller reads it.
|
|
func named(c CheckBuilt) []NamedBuild {
|
|
n := NamedBuild{Pull: aPull, Check: "build-1", Module: c.Module, Key: c.Key, Digests: map[string]string{}}
|
|
for _, m := range c.Made {
|
|
d := m.Digest
|
|
if d == "" {
|
|
d = DigestOf(m.Reference)
|
|
}
|
|
n.Digests[m.Name] = d
|
|
}
|
|
return []NamedBuild{n}
|
|
}
|
|
|
|
func withVersion(t *testing.T, v string) {
|
|
t.Helper()
|
|
was := Version
|
|
Version = v
|
|
t.Cleanup(func() { Version = was })
|
|
}
|
|
|
|
// A merge whose reuse key equals its check's builds nothing again: the walk copies the check's build by digest
|
|
// and registers that digest.
|
|
func TestAWalkWhoseKeyIsItsChecksBuildsNothingAndRegistersTheChecksDigest(t *testing.T) {
|
|
withVersion(t, "builder-1")
|
|
scratch := newScratch()
|
|
checked := aCheckBuilt(t, scratch, "tree-one")
|
|
|
|
r, run, workspace := aModuleAt(t, "tree-one")
|
|
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
|
|
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if dockerBuilt(r) || len(r.images) > 0 || len(r.archives) > 0 {
|
|
t.Fatalf("the walk built again: ran %v, published %v %v", r.ran, r.images, r.archives)
|
|
}
|
|
if got.Reused == nil || got.Why != "" {
|
|
t.Fatalf("the walk did not say it reused the check's build: reused %+v, why %q", got.Reused, got.Why)
|
|
}
|
|
if got.ReuseKey != checked.Key {
|
|
t.Fatalf("the walk's key %s is not the check's %s", got.ReuseKey, checked.Key)
|
|
}
|
|
if len(scratch.took) != 2 {
|
|
t.Fatalf("the walk copied %v, not the check's two artifacts", scratch.took)
|
|
}
|
|
for _, made := range checked.Made {
|
|
registered := strings.Replace(made.Reference, ScratchPrefix, "", 1)
|
|
found := false
|
|
for _, b := range got.Built {
|
|
found = found || (b.Reference == registered && b.Digest == made.Digest)
|
|
}
|
|
if !found {
|
|
t.Fatalf("the walk registers %+v, not the check's %s copied", got.Built, made.Reference)
|
|
}
|
|
}
|
|
if image, _ := got.Manifest.Resources[0]["image"].(string); image != "registry.invalid/meshboard/server@sha256:"+strings.Repeat("a", 64) {
|
|
t.Fatalf("the manifest pins %q, not the check's digest in the module's namespace", image)
|
|
}
|
|
words := got.Reused.Words()
|
|
if !strings.HasPrefix(words, "reused from the check of novox/meshboard#7 at head abcdefabcdef (merged into trunk 123456123456)") ||
|
|
!strings.Contains(words, "sha256:") {
|
|
t.Fatalf("the walk says %q", words)
|
|
}
|
|
}
|
|
|
|
// A record no verdict names is refused and built instead, and the walk says so: the store takes writes from anything
|
|
// on its network, so a record planted under another merge's key is never registered.
|
|
func TestARecordNoVerdictNamesIsRefusedAndBuilt(t *testing.T) {
|
|
withVersion(t, "builder-1")
|
|
scratch := newScratch()
|
|
checked := aCheckBuilt(t, scratch, "tree-one")
|
|
for name, leave := range map[string][]NamedBuild{
|
|
"no verdict": nil,
|
|
"another digest": func() []NamedBuild {
|
|
n := named(checked)
|
|
n[0].Digests["server"] = "sha256:" + strings.Repeat("e", 64)
|
|
return n
|
|
}(),
|
|
"another check": func() []NamedBuild { n := named(checked); n[0].Check = "build-9"; return n }(),
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
scratch.took = nil
|
|
r, run, workspace := aModuleAt(t, "tree-one")
|
|
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: leave},
|
|
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.Reused != nil || len(scratch.took) > 0 || !dockerBuilt(r) {
|
|
t.Fatalf("a record no verdict names was taken: %+v, copied %v", got.Reused, scratch.took)
|
|
}
|
|
if !strings.Contains(got.Why, "no check verdict of the walk's pull requests names it") {
|
|
t.Fatalf("the walk says %q", got.Why)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A differing key builds, and says which part differs.
|
|
func TestAWalkWhoseKeyDiffersBuildsAndSaysWhy(t *testing.T) {
|
|
withVersion(t, "builder-1")
|
|
scratch := newScratch()
|
|
checked := aCheckBuilt(t, scratch, "tree-one")
|
|
|
|
r, run, workspace := aModuleAt(t, "tree-two")
|
|
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
|
|
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !dockerBuilt(r) || got.Reused != nil {
|
|
t.Fatalf("a walk whose source differs did not build: reused %+v", got.Reused)
|
|
}
|
|
if !strings.Contains(got.Why, "the key differs") || !strings.Contains(got.Why, partTree) {
|
|
t.Fatalf("the walk says %q, not which part of the key differs", got.Why)
|
|
}
|
|
|
|
// And a builder of another version builds too, saying so.
|
|
withVersion(t, "builder-2")
|
|
r, run, workspace = aModuleAt(t, "tree-one")
|
|
got, err = BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
|
|
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !dockerBuilt(r) || !strings.Contains(got.Why, partBuilder) {
|
|
t.Fatalf("a newer builder took the older one's build, or did not say why not: %q", got.Why)
|
|
}
|
|
|
|
// No check of the walk's pull requests built it: said.
|
|
r, run, workspace = aModuleAt(t, "tree-one")
|
|
got, _ = BuildReusing(t.Context(), run, r, Reuse{Scratch: newScratch(), Pulls: []PullRef{aPull}},
|
|
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
|
if !dockerBuilt(r) || !strings.Contains(got.Why, "no check of novox/meshboard#7 built it") {
|
|
t.Fatalf("the walk says %q", got.Why)
|
|
}
|
|
}
|
|
|
|
// A copy that fails is said and falls back to a build; it never fails the walk.
|
|
func TestAFailedCopyOfAChecksBuildFallsBackToBuilding(t *testing.T) {
|
|
withVersion(t, "builder-1")
|
|
scratch := newScratch()
|
|
checked := aCheckBuilt(t, scratch, "tree-one")
|
|
scratch.failTake = true
|
|
|
|
r, run, workspace := aModuleAt(t, "tree-one")
|
|
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
|
|
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
|
if err != nil {
|
|
t.Fatalf("a failed copy failed the walk: %v", err)
|
|
}
|
|
if !dockerBuilt(r) || got.Reused != nil || len(r.images) == 0 {
|
|
t.Fatal("a failed copy did not fall back to a build")
|
|
}
|
|
if !strings.Contains(got.Why, "copying meshboard's") || !strings.Contains(got.Why, "refused the mount") {
|
|
t.Fatalf("the walk says %q", got.Why)
|
|
}
|
|
for repository := range r.images {
|
|
if strings.HasPrefix(repository, ScratchPrefix) {
|
|
t.Fatalf("the walk's own build went into the scratch namespace: %s", repository)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Only a walk takes a check's build: a plain build — a check's, a dry run's, `build --ref`'s — builds, and a commit
|
|
// off the trunk never takes one.
|
|
func TestOnlyAWalkOfATrunkCommitTakesAChecksBuild(t *testing.T) {
|
|
withVersion(t, "builder-1")
|
|
scratch := newScratch()
|
|
checked := aCheckBuilt(t, scratch, "tree-one")
|
|
|
|
r, run, workspace := aModuleAt(t, "tree-one")
|
|
if _, err := Build(t.Context(), run, r, "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil,
|
|
Npmrc{}, GitCredential{}, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !dockerBuilt(r) || len(scratch.took) > 0 {
|
|
t.Fatal("a build not asked to reuse took a check's build")
|
|
}
|
|
|
|
r, _, workspace = aModuleAt(t, "tree-one") // no trunk said: off it
|
|
got, err := BuildReusing(t.Context(), r.run, r, Reuse{Scratch: scratch, Named: named(checked)}, "https://forge.invalid/meshboard.git",
|
|
"mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !dockerBuilt(r) || got.Reused != nil || !strings.Contains(got.Why, "not on its trunk") {
|
|
t.Fatalf("a commit off the trunk took a check's build: %q", got.Why)
|
|
}
|
|
}
|
|
|
|
// A check that did not pass builds nothing for its merge, and one asked to build nothing builds nothing.
|
|
func TestACheckThatFailedBuildsNothingForItsMerge(t *testing.T) {
|
|
if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "fail"}}) == "" {
|
|
t.Fatal("a check whose repository layer failed builds for its merge")
|
|
}
|
|
if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "warning"}}) != "" {
|
|
t.Fatal("a check that passed with a warning builds nothing for its merge")
|
|
}
|
|
r, run, workspace := aModuleAt(t, "tree-one")
|
|
if built := buildForTheMerge(t.Context(), run, CheckSpec{ID: "build-1"}, CheckBuilding{Publish: r, Scratch: newScratch()},
|
|
workspace, "", "", CheckVerdict{}, GitCredential{}, nil, logging(nil)); built != nil || len(r.ran) > 0 {
|
|
t.Fatalf("a check moving no module built %v, ran %v", built, r.ran)
|
|
}
|
|
}
|
|
|
|
// A module whose source does not pin its build has no key and is not built by the check; what stands after it
|
|
// is not built either.
|
|
func TestACheckBuildsNoModuleWithoutAKeyNorWhatStandsAfterIt(t *testing.T) {
|
|
withVersion(t, "builder-1")
|
|
r, run, workspace := aModuleAt(t, "tree-one")
|
|
r.contents["mod/"+ManifestName] = `{"module":"meshboard","version":"1","build":{"artifacts":[
|
|
{"name":"mirror","kind":"upstream","from":"docker.io/library/alpine:3"}]}}`
|
|
spec := CheckSpec{ID: "build-1", Owner: "novox", Repo: "meshboard", Number: 7, Builds: []CheckBuildSpec{
|
|
{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", Tier: 0},
|
|
{Module: "after", Repository: "https://forge.invalid/after.git", Tier: 1}}}
|
|
built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: newScratch()}, workspace, "", "",
|
|
CheckVerdict{}, GitCredential{}, nil, logging(nil))
|
|
if len(built) != 2 || !strings.Contains(built[0].NotBuilt, "named by tag") ||
|
|
!strings.Contains(built[1].NotBuilt, "stands after meshboard") {
|
|
t.Fatalf("built %+v", built)
|
|
}
|
|
if len(r.images) > 0 {
|
|
t.Fatalf("a module without a key was published: %v", r.images)
|
|
}
|
|
}
|
|
|
|
// What the key names: every input a fresh build could differ by.
|
|
func TestTheReuseKeyNamesEveryInputOfTheBuild(t *testing.T) {
|
|
src := newSourceInputs("m")
|
|
src.tree = "t1"
|
|
src.bases = []string{"store:5000/b/x@sha256:" + strings.Repeat("b", 64)}
|
|
src.contexts["server"] = "ctx1"
|
|
src.toolchains["tools"] = "go sha256:c 1234"
|
|
manifest := catalogue.Manifest{Module: "m"}
|
|
base := keyOf(src, manifest, "v1")
|
|
if base.Key == "" {
|
|
t.Fatalf("no key: %s", base.Why)
|
|
}
|
|
for name, change := range map[string]func(*sourceInputs) string{
|
|
partTree: func(s *sourceInputs) string { s.tree = "t2"; return "v1" },
|
|
partBases: func(s *sourceInputs) string {
|
|
s.bases = []string{"other@sha256:" + strings.Repeat("c", 64)}
|
|
return "v1"
|
|
},
|
|
partContext: func(s *sourceInputs) string { s.contexts["server"] = "ctx2"; return "v1" },
|
|
partTool: func(s *sourceInputs) string { s.toolchains["tools"] = "go sha256:d 1234"; return "v1" },
|
|
partBuilder: func(*sourceInputs) string { return "v2" },
|
|
} {
|
|
s := src.copy()
|
|
version := change(s)
|
|
if k := keyOf(s, manifest, version); k.Key == base.Key {
|
|
t.Errorf("a change of %s left the key the same", name)
|
|
}
|
|
}
|
|
// A base copied to another address is the same base: the key names it by digest.
|
|
moved := src.copy()
|
|
moved.bases = []string{"elsewhere:5000/scratch/b/x@sha256:" + strings.Repeat("b", 64)}
|
|
if keyOf(moved, manifest, "v1").Key != base.Key {
|
|
t.Error("the same base at another address changed the key")
|
|
}
|
|
if k := keyOf(src, manifest, ""); k.Key != "" {
|
|
t.Error("a builder whose version cannot be said keyed its build")
|
|
}
|
|
unpinned := src.copy()
|
|
unpinned.notPinned("it resolves packages from the mesh's registry at build time")
|
|
if k := keyOf(unpinned, manifest, "v1"); k.Key != "" || !strings.Contains(k.Why, "registry") {
|
|
t.Errorf("a build its source does not pin was keyed: %+v", k)
|
|
}
|
|
}
|
|
|
|
// aTaggingRegistry is the in-memory registry with tags and their listing, as the store's scratch namespace needs.
|
|
type aTaggingRegistry struct {
|
|
*aRegistryOfRepositories
|
|
tags map[string]map[string]string // repository → tag → manifest digest
|
|
}
|
|
|
|
func (m *aTaggingRegistry) serve() http.Handler {
|
|
under := m.handler()
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if repository, ok := strings.CutSuffix(strings.TrimPrefix(r.URL.Path, "/v2/"), "/tags/list"); ok {
|
|
m.mu.Lock()
|
|
var tags []string
|
|
for tag := range m.tags[repository] {
|
|
tags = append(tags, tag)
|
|
}
|
|
m.mu.Unlock()
|
|
if tags == nil {
|
|
w.WriteHeader(http.StatusNotFound)
|
|
return
|
|
}
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"name": repository, "tags": tags})
|
|
return
|
|
}
|
|
repository, kind, rest := splitPath(r.URL.Path)
|
|
if kind == "blobs" && r.Method == http.MethodHead {
|
|
m.mu.Lock()
|
|
body, linked := m.blobs[rest], m.links[repository][rest]
|
|
m.mu.Unlock()
|
|
if !linked {
|
|
w.WriteHeader(http.StatusNotFound)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
|
w.WriteHeader(http.StatusOK)
|
|
return
|
|
}
|
|
if kind != "manifests" {
|
|
under.ServeHTTP(w, r)
|
|
return
|
|
}
|
|
if !strings.HasPrefix(rest, "sha256:") {
|
|
m.mu.Lock()
|
|
switch r.Method {
|
|
case http.MethodPut:
|
|
body, _ := readAll(r)
|
|
if m.tags[repository] == nil {
|
|
m.tags[repository] = map[string]string{}
|
|
}
|
|
m.tags[repository][rest] = digestOf(body)
|
|
m.manifests[repository+"@"+digestOf(body)] = body
|
|
m.mu.Unlock()
|
|
w.WriteHeader(http.StatusCreated)
|
|
return
|
|
default:
|
|
digest, ok := m.tags[repository][rest]
|
|
body, held := m.manifests[repository+"@"+digest]
|
|
m.mu.Unlock()
|
|
if !ok || !held {
|
|
w.WriteHeader(http.StatusNotFound)
|
|
return
|
|
}
|
|
w.Header().Set("Docker-Content-Digest", digest)
|
|
w.Header().Set("Content-Type", mediaTypeOf(body))
|
|
if r.Method == http.MethodGet {
|
|
_, _ = w.Write(body)
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusOK)
|
|
return
|
|
}
|
|
}
|
|
if r.Method == http.MethodDelete {
|
|
m.mu.Lock()
|
|
for _, tags := range m.tags {
|
|
for tag, digest := range tags {
|
|
if digest == rest {
|
|
delete(tags, tag)
|
|
}
|
|
}
|
|
}
|
|
m.mu.Unlock()
|
|
}
|
|
under.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
|
|
// A check's builds go after seven days, and when its pull request closes or is merged; the walk's copy in the
|
|
// module's own namespace stays.
|
|
func TestCheckBuildsExpireAfterSevenDaysAndGoWhenTheirPullRequestCloses(t *testing.T) {
|
|
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
|
|
server := httptest.NewServer(m.serve())
|
|
defer server.Close()
|
|
address := strings.TrimPrefix(server.URL, "http://")
|
|
registry := Registry{Address: address}
|
|
ctx := t.Context()
|
|
now := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
|
|
|
|
image := func(repository, body string) string {
|
|
digest := digestOf([]byte(body))
|
|
m.manifests[repository+"@"+digest] = []byte(body)
|
|
return address + "/" + repository + "@" + digest
|
|
}
|
|
check := func(pull PullRef, id string, at time.Time, key string) string {
|
|
reference := image("scratch/meshboard/server", `{"mediaType":"`+mediaManifestOCI+`","layers":[],"n":"`+id+`"}`)
|
|
build := CheckBuild{Key: key, Module: "meshboard", Made: []catalogue.Built{{Name: "server", Kind: "image",
|
|
Reference: reference}}, Pull: pull, Check: id, At: at}
|
|
if err := registry.Keep(ctx, pull, id, at, []CheckBuild{build}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return reference
|
|
}
|
|
held := func(reference string) bool {
|
|
_, digest, _ := strings.Cut(reference, "@")
|
|
_, ok := m.manifests["scratch/meshboard/server@"+digest]
|
|
return ok
|
|
}
|
|
other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8}
|
|
old := check(aPull, "build-1", now.Add(-8*24*time.Hour), "r1:"+strings.Repeat("1", 64))
|
|
fresh := check(aPull, "build-2", now.Add(-time.Hour), "r1:"+strings.Repeat("2", 64))
|
|
others := check(other, "build-3", now.Add(-time.Hour), "r1:"+strings.Repeat("3", 64))
|
|
walked := image("meshboard/server", `{"mediaType":"`+mediaManifestOCI+`","layers":[],"n":"walk"}`)
|
|
|
|
if _, found, err := registry.Find(ctx, "meshboard", "r1:"+strings.Repeat("2", 64)); err != nil || !found {
|
|
t.Fatalf("a check build kept is not found by its key: %v %v", found, err)
|
|
}
|
|
n, err := registry.Expire(ctx, CheckBuildsKept, now)
|
|
if err != nil || n != 1 {
|
|
t.Fatalf("expired %d checks (%v), want the one eight days old", n, err)
|
|
}
|
|
if held(old) || !held(fresh) || !held(others) {
|
|
t.Fatalf("after the week's sweep: old %v, fresh %v, another's %v", held(old), held(fresh), held(others))
|
|
}
|
|
if _, found, _ := registry.Find(ctx, "meshboard", "r1:"+strings.Repeat("1", 64)); found {
|
|
t.Fatal("the expired check build's record is still found")
|
|
}
|
|
|
|
n, err = registry.Forget(ctx, aPull)
|
|
if err != nil || n != 1 {
|
|
t.Fatalf("forgot %d checks of %s (%v)", n, aPull, err)
|
|
}
|
|
if held(fresh) || !held(others) {
|
|
t.Fatalf("after %s closed: its build held %v, another pull request's %v", aPull, held(fresh), held(others))
|
|
}
|
|
if _, ok := m.manifests["meshboard/server@"+strings.SplitN(walked, "@", 2)[1]]; !ok {
|
|
t.Fatal("letting go of check builds took the walk's copy in the module's namespace")
|
|
}
|
|
if left, _ := registry.OfPulls(ctx, []PullRef{aPull}); len(left) != 0 {
|
|
t.Fatalf("%s still has check builds: %+v", aPull, left)
|
|
}
|
|
}
|
|
|
|
// The walk copies a check build by digest: the digest registered is the digest the check built, and the copy is
|
|
// in the module's own namespace.
|
|
func TestAChecksBuildIsCopiedByDigestIntoTheModulesNamespace(t *testing.T) {
|
|
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
|
|
server := httptest.NewServer(m.serve())
|
|
defer server.Close()
|
|
address := strings.TrimPrefix(server.URL, "http://")
|
|
registry := Registry{Address: address}
|
|
|
|
layer := []byte("layer")
|
|
m.blobs[digestOf(layer)] = layer
|
|
m.link("scratch/meshboard/server", digestOf(layer))
|
|
manifest := []byte(`{"mediaType":"` + mediaManifestOCI + `","layers":[{"digest":"` + digestOf(layer) + `","size":5}]}`)
|
|
m.manifests["scratch/meshboard/server@"+digestOf(manifest)] = manifest
|
|
archive := []byte("archive")
|
|
m.blobs[digestOf(archive)] = archive
|
|
m.link("scratch/meshboard/look", digestOf(archive))
|
|
|
|
got, err := registry.Take(t.Context(), "meshboard", catalogue.Built{Name: "server", Kind: "image",
|
|
Reference: address + "/scratch/meshboard/server@" + digestOf(manifest)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.Reference != address+"/meshboard/server@"+digestOf(manifest) {
|
|
t.Fatalf("copied as %s", got.Reference)
|
|
}
|
|
if m.uploads != 0 || m.mounts != 1 {
|
|
t.Fatalf("the image's layer was moved (%d uploads, %d mounts), not mounted", m.uploads, m.mounts)
|
|
}
|
|
got, err = registry.Take(t.Context(), "meshboard", catalogue.Built{Name: "look", Kind: "archive",
|
|
Digest: digestOf(archive), Reference: "http://" + address + "/v2/scratch/meshboard/look/blobs/" + digestOf(archive)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.Digest != digestOf(archive) || got.Reference != "http://"+address+"/v2/meshboard/look/blobs/"+digestOf(archive) ||
|
|
!m.links["meshboard/look"][digestOf(archive)] {
|
|
t.Fatalf("the archive was copied as %+v", got)
|
|
}
|
|
}
|
|
|
|
// Two checks that built one digest in one repository share it: letting go of one check's builds keeps it while the
|
|
// other still names it, and the last to go takes it.
|
|
func TestABuildTwoChecksShareGoesWithTheLastOfThem(t *testing.T) {
|
|
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
|
|
server := httptest.NewServer(m.serve())
|
|
defer server.Close()
|
|
address := strings.TrimPrefix(server.URL, "http://")
|
|
registry := Registry{Address: address}
|
|
ctx := t.Context()
|
|
now := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
|
|
|
|
body := `{"mediaType":"` + mediaManifestOCI + `","layers":[]}`
|
|
digest := digestOf([]byte(body))
|
|
m.manifests["scratch/meshboard/server@"+digest] = []byte(body)
|
|
reference := address + "/scratch/meshboard/server@" + digest
|
|
other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8}
|
|
for i, pull := range []PullRef{aPull, other} {
|
|
id := "build-" + strconv.Itoa(i+1)
|
|
b := CheckBuild{Key: "r1:" + strings.Repeat(strconv.Itoa(i+1), 64), Module: "meshboard", Pull: pull, Check: id,
|
|
At: now, Made: []catalogue.Built{{Name: "server", Kind: "image", Reference: reference}}}
|
|
if err := registry.Keep(ctx, pull, id, now, []CheckBuild{b}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
held := func() bool { _, ok := m.manifests["scratch/meshboard/server@"+digest]; return ok }
|
|
|
|
if n, err := registry.Forget(ctx, aPull); err != nil || n != 1 {
|
|
t.Fatalf("forgot %d (%v)", n, err)
|
|
}
|
|
if !held() {
|
|
t.Fatal("letting go of one check's builds took a build another check still names")
|
|
}
|
|
if n, err := registry.Forget(ctx, other); err != nil || n != 1 {
|
|
t.Fatalf("forgot %d (%v)", n, err)
|
|
}
|
|
if held() {
|
|
t.Fatal("the build stayed after the last check naming it went")
|
|
}
|
|
}
|
|
|
|
// aCheckAgain is a later check (build-2) of aPull building the module from a tree, with the builds the build seat's
|
|
// verdicts name for it (novox/hq issue 703).
|
|
func aCheckAgain(t *testing.T, scratch *aScratch, tree string, leave []NamedBuild) (CheckBuilt, *recorded) {
|
|
t.Helper()
|
|
return aCheckOf(t, scratch, aPull, "abcdefabcdef", tree, leave)
|
|
}
|
|
|
|
// aCheckOf is a later check (build-2) of a pull request at a head, the trunk moved since build-1.
|
|
func aCheckOf(t *testing.T, scratch *aScratch, pull PullRef, head, tree string, leave []NamedBuild) (CheckBuilt, *recorded) {
|
|
t.Helper()
|
|
r, run, workspace := aModuleAt(t, tree)
|
|
spec := CheckSpec{ID: "build-2", Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number,
|
|
Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod",
|
|
Ref: "main"}}}
|
|
built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: scratch, Named: leave},
|
|
workspace, "", "", CheckVerdict{Head: head, Trunk: "654321654321"}, GitCredential{}, nil, logging(nil))
|
|
if len(built) != 1 || built[0].NotBuilt != "" || built[0].Key == "" {
|
|
t.Fatalf("the second check did not answer the module for its merge: %+v", built)
|
|
}
|
|
return built[0], r
|
|
}
|
|
|
|
// A check asked again when only the trunk moved, whose key is unchanged, builds nothing: it keeps the build an
|
|
// earlier verdict names and names it by key and digest, so the walk can take it (novox/hq issue 703).
|
|
func TestASecondCheckWhoseKeyIsUnchangedBuildsNothingAndNamesTheEarlierBuild(t *testing.T) {
|
|
withVersion(t, "builder-1")
|
|
scratch := newScratch()
|
|
first := aCheckBuilt(t, scratch, "tree-one")
|
|
|
|
again, r := aCheckAgain(t, scratch, "tree-one", named(first))
|
|
if dockerBuilt(r) || len(r.images) > 0 || len(r.archives) > 0 {
|
|
t.Fatalf("the second check built again: ran %v, published %v %v", r.ran, r.images, r.archives)
|
|
}
|
|
if again.Key != first.Key || again.Kept != "build-1" || !reflect.DeepEqual(again.Made, first.Made) {
|
|
t.Fatalf("the second check names %+v, not the first check's build %+v", again, first)
|
|
}
|
|
// The walk takes it by what the second verdict names.
|
|
leave := named(again)
|
|
leave[0].Check = again.Kept
|
|
w, run, workspace := aModuleAt(t, "tree-one")
|
|
got, err := BuildReusing(t.Context(), run, w, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: leave},
|
|
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
|
|
if err != nil || got.Reused == nil || dockerBuilt(w) {
|
|
t.Fatalf("the walk did not take the kept build: %v, why %q", err, got.Why)
|
|
}
|
|
}
|
|
|
|
// A check whose key changed builds, as before.
|
|
func TestASecondCheckWhoseKeyChangedBuilds(t *testing.T) {
|
|
withVersion(t, "builder-1")
|
|
scratch := newScratch()
|
|
first := aCheckBuilt(t, scratch, "tree-one")
|
|
|
|
again, r := aCheckAgain(t, scratch, "tree-two", named(first))
|
|
if !dockerBuilt(r) || again.Kept != "" || again.Key == first.Key {
|
|
t.Fatalf("a check whose key changed did not build: %+v", again)
|
|
}
|
|
}
|
|
|
|
// A scratch record no verdict names is never kept: the store takes writes from anything on its network, so a
|
|
// check builds rather than name a planted record (novox/hq issue 703, ADR 0350 §2).
|
|
func TestASecondCheckKeepsNoRecordAVerdictDoesNotName(t *testing.T) {
|
|
withVersion(t, "builder-1")
|
|
scratch := newScratch()
|
|
first := aCheckBuilt(t, scratch, "tree-one")
|
|
for name, leave := range map[string][]NamedBuild{
|
|
"no verdict": nil,
|
|
"another digest": func() []NamedBuild {
|
|
n := named(first)
|
|
n[0].Digests["server"] = "sha256:" + strings.Repeat("e", 64)
|
|
return n
|
|
}(),
|
|
"another key": func() []NamedBuild { n := named(first); n[0].Key = "r1:" + strings.Repeat("f", 64); return n }(),
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
again, r := aCheckAgain(t, scratch, "tree-one", leave)
|
|
if !dockerBuilt(r) || again.Kept != "" {
|
|
t.Fatalf("a record no verdict names was kept: %+v", again)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A build a later check kept stays when the check that built it goes: its artifacts and its record, which the later
|
|
// check's verdict names (novox/hq issue 703, ADR 0350 §5).
|
|
func TestABuildALaterCheckKeptStaysWhenTheCheckThatBuiltItGoes(t *testing.T) {
|
|
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
|
|
server := httptest.NewServer(m.serve())
|
|
defer server.Close()
|
|
address := strings.TrimPrefix(server.URL, "http://")
|
|
registry := Registry{Address: address}
|
|
ctx := t.Context()
|
|
now := time.Date(2026, 10, 14, 9, 0, 0, 0, time.UTC)
|
|
|
|
body := `{"mediaType":"` + mediaManifestOCI + `","layers":[]}`
|
|
digest := digestOf([]byte(body))
|
|
m.manifests["scratch/meshboard/server@"+digest] = []byte(body)
|
|
key := "r1:" + strings.Repeat("1", 64)
|
|
b := CheckBuild{Key: key, Module: "meshboard", Pull: aPull, Check: "build-1", At: now,
|
|
Made: []catalogue.Built{{Name: "server", Kind: "image", Reference: address + "/scratch/meshboard/server@" + digest}}}
|
|
if err := registry.Keep(ctx, aPull, "build-1", now, []CheckBuild{b}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Another pull request's check keeps it: its own document names the build, which stays the first check's.
|
|
other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8}
|
|
if err := registry.Keep(ctx, other, "build-2", now, []CheckBuild{b}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
held := func() bool { _, ok := m.manifests["scratch/meshboard/server@"+digest]; return ok }
|
|
found := func() bool { _, ok, _ := registry.Find(ctx, "meshboard", key); return ok }
|
|
|
|
if n, err := registry.Forget(ctx, aPull); err != nil || n != 1 {
|
|
t.Fatalf("forgot %d (%v)", n, err)
|
|
}
|
|
if !held() || !found() {
|
|
t.Fatalf("the build a later check names went with the check that built it: held %v, record %v", held(), found())
|
|
}
|
|
if n, err := registry.Forget(ctx, other); err != nil || n != 1 {
|
|
t.Fatalf("forgot %d (%v)", n, err)
|
|
}
|
|
if held() || found() {
|
|
t.Fatalf("the build stayed after the last check naming it went: held %v, record %v", held(), found())
|
|
}
|
|
}
|
|
|
|
// Only an earlier check of the same pull request at the same head is kept from: another head of it, or another pull
|
|
// request whose verdict names the same key and digests, builds (novox/hq issue 703).
|
|
func TestACheckKeepsNoBuildOfAnotherHeadOrPullRequest(t *testing.T) {
|
|
withVersion(t, "builder-1")
|
|
other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8}
|
|
for name, c := range map[string]struct {
|
|
pull PullRef
|
|
head string
|
|
}{
|
|
"another head": {aPull, "999999999999"},
|
|
"another pull request": {other, "abcdefabcdef"},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
scratch := newScratch()
|
|
first := aCheckBuilt(t, scratch, "tree-one")
|
|
leave := named(first)
|
|
leave[0].Pull = c.pull // as if its own verdict named it
|
|
again, r := aCheckOf(t, scratch, c.pull, c.head, "tree-one", leave)
|
|
if !dockerBuilt(r) || again.Kept != "" {
|
|
t.Fatalf("a build of %s was kept: %+v", name, again)
|
|
}
|
|
})
|
|
}
|
|
}
|