jochen
c5663aa18b
An unrecorded index or a copy in progress can name a platform the records do not see, so only a person's collect, after its dry run, takes an index's platforms, and only once every kept index of each repository it touches was read. A copy missing a platform is copied again, and a copy a build holds again is no longer recorded as collected (review of #144).
376 lines
13 KiB
Go
376 lines
13 KiB
Go
package builder
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
)
|
|
|
|
// One copy per upstream image, whichever modules stand on it (novox/hq ADR 0257).
|
|
|
|
func TestAnUpstreamImageHasOneRepositoryNamedForIt(t *testing.T) {
|
|
for from, want := range map[string]string{
|
|
"golang@sha256:abc": "upstream/docker.io/library/golang",
|
|
"n8nio/n8n@sha256:abc": "upstream/docker.io/n8nio/n8n",
|
|
"quay.io/minio/mc@sha256:abc": "upstream/quay.io/minio/mc",
|
|
"ghcr.io/Mailu/Admin@sha256:abc": "upstream/ghcr.io/mailu/admin",
|
|
"localhost:5000/x/y@sha256:abc": "upstream/localhost-5000/x/y",
|
|
"docker.io/library/alpine:3.20@sha256:ab": "upstream/docker.io/library/alpine",
|
|
} {
|
|
got, err := MirrorRepository(from)
|
|
if err != nil {
|
|
t.Fatalf("%s: %v", from, err)
|
|
}
|
|
if got != want {
|
|
t.Errorf("%s: got %q want %q", from, got, want)
|
|
}
|
|
}
|
|
if got := FormerMirrorRepository("route-proxy", "GO_BASE"); got != "route-proxy/on-go_base" {
|
|
t.Fatalf("the former repository is %q", got)
|
|
}
|
|
}
|
|
|
|
// aRegistryOfRepositories is a registry the way the real one is: blobs stored once, and each
|
|
// repository linking the blobs and manifests it holds. It mounts a blob across repositories.
|
|
type aRegistryOfRepositories struct {
|
|
mu sync.Mutex
|
|
blobs map[string][]byte // digest → bytes, stored once
|
|
links map[string]map[string]bool // repository → blob digests it links
|
|
manifests map[string][]byte // repository@digest → document
|
|
uploads int
|
|
mounts int
|
|
gets int
|
|
// noMount answers every mount with an ordinary upload's location, as a registry that cannot
|
|
// mount across repositories does.
|
|
noMount bool
|
|
}
|
|
|
|
func newRegistryOfRepositories() *aRegistryOfRepositories {
|
|
return &aRegistryOfRepositories{blobs: map[string][]byte{}, links: map[string]map[string]bool{},
|
|
manifests: map[string][]byte{}}
|
|
}
|
|
|
|
func (m *aRegistryOfRepositories) link(repository, digest string) {
|
|
if m.links[repository] == nil {
|
|
m.links[repository] = map[string]bool{}
|
|
}
|
|
m.links[repository][digest] = true
|
|
}
|
|
|
|
// split reads /v2/<repository>/<kind>/<rest> with a repository of any depth.
|
|
func splitPath(path string) (repository, kind, rest string) {
|
|
path = strings.TrimPrefix(path, "/v2/")
|
|
for _, k := range []string{"/manifests/", "/blobs/uploads/", "/blobs/"} {
|
|
if i := strings.Index(path, k); i >= 0 {
|
|
return path[:i], strings.Trim(k, "/"), path[i+len(k):]
|
|
}
|
|
}
|
|
return "", "", ""
|
|
}
|
|
|
|
func (m *aRegistryOfRepositories) handler() http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
repository, kind, rest := splitPath(r.URL.Path)
|
|
switch {
|
|
case kind == "manifests" && (r.Method == http.MethodHead || r.Method == http.MethodGet):
|
|
body, ok := m.manifests[repository+"@"+rest]
|
|
if !ok {
|
|
w.WriteHeader(http.StatusNotFound)
|
|
return
|
|
}
|
|
if r.Method == http.MethodGet {
|
|
m.gets++
|
|
w.Header().Set("Content-Type", mediaTypeOf(body))
|
|
_, _ = w.Write(body)
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusOK)
|
|
case kind == "manifests" && r.Method == http.MethodPut:
|
|
body, _ := readAll(r)
|
|
m.manifests[repository+"@"+digestOf(body)] = body
|
|
w.WriteHeader(http.StatusCreated)
|
|
case kind == "blobs" && (r.Method == http.MethodHead || r.Method == http.MethodGet):
|
|
if !m.links[repository][rest] {
|
|
w.WriteHeader(http.StatusNotFound)
|
|
return
|
|
}
|
|
if r.Method == http.MethodGet {
|
|
m.gets++
|
|
_, _ = w.Write(m.blobs[rest])
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusOK)
|
|
case kind == "manifests" && r.Method == http.MethodDelete:
|
|
delete(m.manifests, repository+"@"+rest)
|
|
w.WriteHeader(http.StatusAccepted)
|
|
case kind == "blobs/uploads" && r.Method == http.MethodPost:
|
|
if digest, from := r.URL.Query().Get("mount"), r.URL.Query().Get("from"); digest != "" && m.links[from][digest] && !m.noMount {
|
|
m.link(repository, digest)
|
|
m.mounts++
|
|
w.WriteHeader(http.StatusCreated)
|
|
return
|
|
}
|
|
w.Header().Set("Location", "/v2/"+repository+"/blobs/uploads/one")
|
|
w.WriteHeader(http.StatusAccepted)
|
|
case kind == "blobs/uploads" && r.Method == http.MethodPut:
|
|
body, _ := readAll(r)
|
|
digest := r.URL.Query().Get("digest")
|
|
if digestOf(body) != digest {
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
return
|
|
}
|
|
m.blobs[digest] = body
|
|
m.link(repository, digest)
|
|
m.uploads++
|
|
w.WriteHeader(http.StatusCreated)
|
|
default:
|
|
w.WriteHeader(http.StatusNotFound)
|
|
}
|
|
})
|
|
}
|
|
|
|
func mediaTypeOf(body []byte) string {
|
|
switch {
|
|
case strings.Contains(string(body), mediaIndexOCI):
|
|
return mediaIndexOCI
|
|
default:
|
|
return mediaManifestOCI
|
|
}
|
|
}
|
|
|
|
// A base a module's own repository already holds is copied into the image's repository from there:
|
|
// every blob mounted, nothing asked of upstream — which may be gone, or rationing anonymous pulls.
|
|
func TestABaseHeldUnderTheModulesFormerRepositoryIsMountedNotFetchedAgain(t *testing.T) {
|
|
src, indexDigest, srcBlobs := anUpstreamRegistry(t)
|
|
dst := newRegistryOfRepositories()
|
|
dstServer := httptest.NewServer(dst.handler())
|
|
defer dstServer.Close()
|
|
address := strings.TrimPrefix(dstServer.URL, "http://")
|
|
r := Registry{Address: address, HTTP: src.Client()}
|
|
host := strings.TrimPrefix(src.URL, "http://")
|
|
|
|
// Before: copied the old way, under the module's repository.
|
|
if _, err := r.MirrorImage(context.Background(), host+"/library/thing:latest", "hello-web/on-thing_base"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
uploaded := dst.uploads
|
|
if uploaded != len(srcBlobs) {
|
|
t.Fatalf("the first copy uploaded %d of %d blobs", uploaded, len(srcBlobs))
|
|
}
|
|
src.Close()
|
|
|
|
from := host + "/library/thing@" + indexDigest
|
|
repository, err := MirrorRepository(from)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
reference, err := r.MirrorBase(context.Background(), from, repository, "hello-web/on-thing_base")
|
|
if err != nil {
|
|
t.Fatalf("a base the registry holds was asked of an upstream that is gone: %v", err)
|
|
}
|
|
if reference != address+"/"+repository+"@"+indexDigest {
|
|
t.Fatalf("pinned as %q", reference)
|
|
}
|
|
if dst.uploads != uploaded {
|
|
t.Fatalf("the move to one repository uploaded %d blobs again", dst.uploads-uploaded)
|
|
}
|
|
if dst.mounts != len(srcBlobs) {
|
|
t.Fatalf("%d of %d blobs mounted", dst.mounts, len(srcBlobs))
|
|
}
|
|
// The index and both platform manifests are in the image's repository, and every blob is linked.
|
|
for key := range dst.manifests {
|
|
if strings.HasPrefix(key, "hello-web/") {
|
|
continue
|
|
}
|
|
if !strings.HasPrefix(key, repository+"@") {
|
|
t.Fatalf("a manifest went to %s", key)
|
|
}
|
|
}
|
|
if n := countPrefix(dst.manifests, repository+"@"); n != 3 {
|
|
t.Fatalf("%d manifests in %s, want the index and two platforms", n, repository)
|
|
}
|
|
for digest := range srcBlobs {
|
|
if !dst.links[repository][digest] {
|
|
t.Fatalf("blob %s is not linked into %s", digest, repository)
|
|
}
|
|
}
|
|
|
|
// A second module standing on the same image: already held, nothing copied, the same reference.
|
|
again, err := r.MirrorBase(context.Background(), from, repository, "other-module/on-thing_base")
|
|
if err != nil || again != reference {
|
|
t.Fatalf("a second module's copy: %q %v", again, err)
|
|
}
|
|
}
|
|
|
|
func countPrefix(m map[string][]byte, prefix string) int {
|
|
n := 0
|
|
for k := range m {
|
|
if strings.HasPrefix(k, prefix) {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
// What a build copied is said whether it worked or not: the copy is in the store either way, and a
|
|
// build that failed after copying is the one record that it is there.
|
|
func TestABuildSaysWhatItMirroredEvenWhenItFails(t *testing.T) {
|
|
// A recipe that reaches for an image nobody declared is refused — after the base was copied.
|
|
r, workspace := aRepository(t, anImage, map[string]string{
|
|
"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}\nCOPY --from=vendor/other:1 /x /x"})
|
|
r.contents["modules/bus/module.json"] = anImage
|
|
r.tree = "aaaa"
|
|
m := &mirroring{recorded: r, at: "registry-a:5000"}
|
|
got, err := Build(context.Background(), r.run, m, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace,
|
|
nil, Npmrc{}, GitCredential{}, nil)
|
|
if err == nil {
|
|
t.Fatal("a recipe fetching an undeclared image was built")
|
|
}
|
|
want := "registry-a:5000/upstream/docker.io/vendor/server@sha256:" + strings.Repeat("1", 64)
|
|
if len(got.Mirrored) != 1 || got.Mirrored[0] != want {
|
|
t.Fatalf("a failed build said it mirrored %v, want [%s]", got.Mirrored, want)
|
|
}
|
|
|
|
// And a build that works says it too.
|
|
r2, workspace2 := aRepository(t, anImage, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"})
|
|
r2.contents["modules/bus/module.json"] = anImage
|
|
r2.tree = "aaaa"
|
|
ok, err := Build(context.Background(), r2.run, &mirroring{recorded: r2, at: "registry-a:5000"},
|
|
"https://forge.invalid/catalogue.git", "modules/bus", "", workspace2, nil, Npmrc{}, GitCredential{}, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(ok.Mirrored) != 1 || ok.Mirrored[0] != want {
|
|
t.Fatalf("a build said it mirrored %v, want [%s]", ok.Mirrored, want)
|
|
}
|
|
}
|
|
|
|
// held puts one image in a repository the old way and answers its index digest and the registry.
|
|
func heldUnderAModule(t *testing.T) (*aRegistryOfRepositories, Registry, string, string, map[string][]byte, func()) {
|
|
t.Helper()
|
|
src, indexDigest, srcBlobs := anUpstreamRegistry(t)
|
|
t.Cleanup(src.Close)
|
|
dst := newRegistryOfRepositories()
|
|
dstServer := httptest.NewServer(dst.handler())
|
|
t.Cleanup(dstServer.Close)
|
|
r := Registry{Address: strings.TrimPrefix(dstServer.URL, "http://"), HTTP: src.Client()}
|
|
host := strings.TrimPrefix(src.URL, "http://")
|
|
if _, err := r.MirrorImage(context.Background(), host+"/library/thing:latest", "hello-web/on-thing_base"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return dst, r, host + "/library/thing@" + indexDigest, indexDigest, srcBlobs, src.Close
|
|
}
|
|
|
|
// An index whose platform manifests are not all held is not "already held": it is copied again, and the
|
|
// copy puts back what is missing — what a sweep that stopped half-way, or ran while a build was copying,
|
|
// leaves behind.
|
|
func TestAnIndexMissingAPlatformIsCopiedAgain(t *testing.T) {
|
|
dst, r, from, indexDigest, _, _ := heldUnderAModule(t)
|
|
repository, _ := MirrorRepository(from)
|
|
if _, err := r.MirrorBase(context.Background(), from, repository, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// A sweep lets one platform go, between this build's copy and the next.
|
|
var platform string
|
|
for key := range dst.manifests {
|
|
if strings.HasPrefix(key, repository+"@") && !strings.HasSuffix(key, indexDigest) {
|
|
platform = key
|
|
break
|
|
}
|
|
}
|
|
delete(dst.manifests, platform)
|
|
if _, err := r.MirrorBase(context.Background(), from, repository, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, back := dst.manifests[platform]; !back {
|
|
t.Fatalf("an index missing %s was taken as held", platform)
|
|
}
|
|
}
|
|
|
|
// The same, for the module's former copy: a former copy missing a platform is not a source; upstream
|
|
// is asked instead.
|
|
func TestAFormerCopyMissingAPlatformIsNotTheSource(t *testing.T) {
|
|
dst, r, from, indexDigest, _, _ := heldUnderAModule(t)
|
|
for key := range dst.manifests {
|
|
if strings.HasPrefix(key, "hello-web/on-thing_base@") && !strings.HasSuffix(key, indexDigest) {
|
|
delete(dst.manifests, key)
|
|
break
|
|
}
|
|
}
|
|
repository, _ := MirrorRepository(from)
|
|
if _, err := r.MirrorBase(context.Background(), from, repository, "hello-web/on-thing_base"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if dst.mounts != 0 {
|
|
t.Fatalf("a former copy missing a platform was mounted from (%d mounts)", dst.mounts)
|
|
}
|
|
if n := countPrefix(dst.manifests, repository+"@"); n != 3 {
|
|
t.Fatalf("%d manifests copied from upstream, want 3", n)
|
|
}
|
|
}
|
|
|
|
// A registry that answers a mount with an upload's location (202) gets the blob moved as before.
|
|
func TestAMountRefusedFallsBackToAnUpload(t *testing.T) {
|
|
dst, r, from, _, srcBlobs, _ := heldUnderAModule(t)
|
|
dst.noMount = true
|
|
uploaded := dst.uploads
|
|
repository, _ := MirrorRepository(from)
|
|
reference, err := r.MirrorBase(context.Background(), from, repository, "hello-web/on-thing_base")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.HasSuffix(reference, repository+"@"+strings.SplitN(from, "@", 2)[1]) {
|
|
t.Fatalf("pinned as %s", reference)
|
|
}
|
|
if dst.mounts != 0 || dst.uploads-uploaded != len(srcBlobs) {
|
|
t.Fatalf("%d mounts, %d uploads; want every blob uploaded from the former copy", dst.mounts, dst.uploads-uploaded)
|
|
}
|
|
for digest := range srcBlobs {
|
|
if !dst.links[repository][digest] {
|
|
t.Fatalf("blob %s is not linked into %s", digest, repository)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A sweep and a build at once: while builds copy the base, platforms are let go of under them. Each
|
|
// build that returns has the copy whole.
|
|
func TestABuildCopyingWhileASweepLetsGoEndsWhole(t *testing.T) {
|
|
dst, r, from, indexDigest, _, _ := heldUnderAModule(t)
|
|
repository, _ := MirrorRepository(from)
|
|
if _, err := r.MirrorBase(context.Background(), from, repository, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
done := make(chan struct{})
|
|
go func() {
|
|
defer close(done)
|
|
for i := 0; i < 50; i++ {
|
|
dst.mu.Lock()
|
|
for key := range dst.manifests {
|
|
if strings.HasPrefix(key, repository+"@") && !strings.HasSuffix(key, indexDigest) {
|
|
delete(dst.manifests, key)
|
|
break
|
|
}
|
|
}
|
|
dst.mu.Unlock()
|
|
}
|
|
}()
|
|
for i := 0; i < 20; i++ {
|
|
if _, err := r.MirrorBase(context.Background(), from, repository, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
<-done
|
|
// The sweep is over; the next build finds what it left and makes the copy whole.
|
|
if _, err := r.MirrorBase(context.Background(), from, repository, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n := countPrefix(dst.manifests, repository+"@"); n != 3 {
|
|
t.Fatalf("after the sweep and the builds, %d manifests in %s, want 3", n, repository)
|
|
}
|
|
}
|