package builder import ( "context" "net/http" "net/http/httptest" "strings" "sync" "testing" ) // One copy per upstream image, whichever modules stand on it (novox/hq ADR 0257). func TestAnUpstreamImageHasOneRepositoryNamedForIt(t *testing.T) { for from, want := range map[string]string{ "golang@sha256:abc": "upstream/docker.io/library/golang", "n8nio/n8n@sha256:abc": "upstream/docker.io/n8nio/n8n", "quay.io/minio/mc@sha256:abc": "upstream/quay.io/minio/mc", "ghcr.io/Mailu/Admin@sha256:abc": "upstream/ghcr.io/mailu/admin", "localhost:5000/x/y@sha256:abc": "upstream/localhost-5000/x/y", "docker.io/library/alpine:3.20@sha256:ab": "upstream/docker.io/library/alpine", } { got, err := MirrorRepository(from) if err != nil { t.Fatalf("%s: %v", from, err) } if got != want { t.Errorf("%s: got %q want %q", from, got, want) } } if got := FormerMirrorRepository("route-proxy", "GO_BASE"); got != "route-proxy/on-go_base" { t.Fatalf("the former repository is %q", got) } } // aRegistryOfRepositories is a registry the way the real one is: blobs stored once, and each // repository linking the blobs and manifests it holds. It mounts a blob across repositories. type aRegistryOfRepositories struct { mu sync.Mutex blobs map[string][]byte // digest → bytes, stored once links map[string]map[string]bool // repository → blob digests it links manifests map[string][]byte // repository@digest → document uploads int mounts int gets int // noMount answers every mount with an ordinary upload's location, as a registry that cannot // mount across repositories does. noMount bool } func newRegistryOfRepositories() *aRegistryOfRepositories { return &aRegistryOfRepositories{blobs: map[string][]byte{}, links: map[string]map[string]bool{}, manifests: map[string][]byte{}} } func (m *aRegistryOfRepositories) link(repository, digest string) { if m.links[repository] == nil { m.links[repository] = map[string]bool{} } m.links[repository][digest] = true } // split reads /v2/// with a repository of any depth. func splitPath(path string) (repository, kind, rest string) { path = strings.TrimPrefix(path, "/v2/") for _, k := range []string{"/manifests/", "/blobs/uploads/", "/blobs/"} { if i := strings.Index(path, k); i >= 0 { return path[:i], strings.Trim(k, "/"), path[i+len(k):] } } return "", "", "" } func (m *aRegistryOfRepositories) handler() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { m.mu.Lock() defer m.mu.Unlock() repository, kind, rest := splitPath(r.URL.Path) switch { case kind == "manifests" && (r.Method == http.MethodHead || r.Method == http.MethodGet): body, ok := m.manifests[repository+"@"+rest] if !ok { w.WriteHeader(http.StatusNotFound) return } if r.Method == http.MethodGet { m.gets++ w.Header().Set("Content-Type", mediaTypeOf(body)) _, _ = w.Write(body) return } w.WriteHeader(http.StatusOK) case kind == "manifests" && r.Method == http.MethodPut: body, _ := readAll(r) m.manifests[repository+"@"+digestOf(body)] = body w.WriteHeader(http.StatusCreated) case kind == "blobs" && (r.Method == http.MethodHead || r.Method == http.MethodGet): if !m.links[repository][rest] { w.WriteHeader(http.StatusNotFound) return } if r.Method == http.MethodGet { m.gets++ _, _ = w.Write(m.blobs[rest]) return } w.WriteHeader(http.StatusOK) case kind == "manifests" && r.Method == http.MethodDelete: delete(m.manifests, repository+"@"+rest) w.WriteHeader(http.StatusAccepted) case kind == "blobs/uploads" && r.Method == http.MethodPost: if digest, from := r.URL.Query().Get("mount"), r.URL.Query().Get("from"); digest != "" && m.links[from][digest] && !m.noMount { m.link(repository, digest) m.mounts++ w.WriteHeader(http.StatusCreated) return } w.Header().Set("Location", "/v2/"+repository+"/blobs/uploads/one") w.WriteHeader(http.StatusAccepted) case kind == "blobs/uploads" && r.Method == http.MethodPut: body, _ := readAll(r) digest := r.URL.Query().Get("digest") if digestOf(body) != digest { w.WriteHeader(http.StatusBadRequest) return } m.blobs[digest] = body m.link(repository, digest) m.uploads++ w.WriteHeader(http.StatusCreated) default: w.WriteHeader(http.StatusNotFound) } }) } func mediaTypeOf(body []byte) string { switch { case strings.Contains(string(body), mediaIndexOCI): return mediaIndexOCI default: return mediaManifestOCI } } // A base a module's own repository already holds is copied into the image's repository from there: // every blob mounted, nothing asked of upstream — which may be gone, or rationing anonymous pulls. func TestABaseHeldUnderTheModulesFormerRepositoryIsMountedNotFetchedAgain(t *testing.T) { src, indexDigest, srcBlobs := anUpstreamRegistry(t) dst := newRegistryOfRepositories() dstServer := httptest.NewServer(dst.handler()) defer dstServer.Close() address := strings.TrimPrefix(dstServer.URL, "http://") r := Registry{Address: address, HTTP: src.Client()} host := strings.TrimPrefix(src.URL, "http://") // Before: copied the old way, under the module's repository. if _, err := r.MirrorImage(context.Background(), host+"/library/thing:latest", "hello-web/on-thing_base"); err != nil { t.Fatal(err) } uploaded := dst.uploads if uploaded != len(srcBlobs) { t.Fatalf("the first copy uploaded %d of %d blobs", uploaded, len(srcBlobs)) } src.Close() from := host + "/library/thing@" + indexDigest repository, err := MirrorRepository(from) if err != nil { t.Fatal(err) } reference, err := r.MirrorBase(context.Background(), from, repository, "hello-web/on-thing_base") if err != nil { t.Fatalf("a base the registry holds was asked of an upstream that is gone: %v", err) } if reference != address+"/"+repository+"@"+indexDigest { t.Fatalf("pinned as %q", reference) } if dst.uploads != uploaded { t.Fatalf("the move to one repository uploaded %d blobs again", dst.uploads-uploaded) } if dst.mounts != len(srcBlobs) { t.Fatalf("%d of %d blobs mounted", dst.mounts, len(srcBlobs)) } // The index and both platform manifests are in the image's repository, and every blob is linked. for key := range dst.manifests { if strings.HasPrefix(key, "hello-web/") { continue } if !strings.HasPrefix(key, repository+"@") { t.Fatalf("a manifest went to %s", key) } } if n := countPrefix(dst.manifests, repository+"@"); n != 3 { t.Fatalf("%d manifests in %s, want the index and two platforms", n, repository) } for digest := range srcBlobs { if !dst.links[repository][digest] { t.Fatalf("blob %s is not linked into %s", digest, repository) } } // A second module standing on the same image: already held, nothing copied, the same reference. again, err := r.MirrorBase(context.Background(), from, repository, "other-module/on-thing_base") if err != nil || again != reference { t.Fatalf("a second module's copy: %q %v", again, err) } } func countPrefix(m map[string][]byte, prefix string) int { n := 0 for k := range m { if strings.HasPrefix(k, prefix) { n++ } } return n } // What a build copied is said whether it worked or not: the copy is in the store either way, and a // build that failed after copying is the one record that it is there. func TestABuildSaysWhatItMirroredEvenWhenItFails(t *testing.T) { // A recipe that reaches for an image nobody declared is refused — after the base was copied. r, workspace := aRepository(t, anImage, map[string]string{ "modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}\nCOPY --from=vendor/other:1 /x /x"}) r.contents["modules/bus/module.json"] = anImage r.tree = "aaaa" m := &mirroring{recorded: r, at: "registry-a:5000"} got, err := Build(context.Background(), r.run, m, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace, nil, Npmrc{}, GitCredential{}, nil) if err == nil { t.Fatal("a recipe fetching an undeclared image was built") } want := "registry-a:5000/upstream/docker.io/vendor/server@sha256:" + strings.Repeat("1", 64) if len(got.Mirrored) != 1 || got.Mirrored[0] != want { t.Fatalf("a failed build said it mirrored %v, want [%s]", got.Mirrored, want) } // And a build that works says it too. r2, workspace2 := aRepository(t, anImage, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"}) r2.contents["modules/bus/module.json"] = anImage r2.tree = "aaaa" ok, err := Build(context.Background(), r2.run, &mirroring{recorded: r2, at: "registry-a:5000"}, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace2, nil, Npmrc{}, GitCredential{}, nil) if err != nil { t.Fatal(err) } if len(ok.Mirrored) != 1 || ok.Mirrored[0] != want { t.Fatalf("a build said it mirrored %v, want [%s]", ok.Mirrored, want) } } // held puts one image in a repository the old way and answers its index digest and the registry. func heldUnderAModule(t *testing.T) (*aRegistryOfRepositories, Registry, string, string, map[string][]byte, func()) { t.Helper() src, indexDigest, srcBlobs := anUpstreamRegistry(t) t.Cleanup(src.Close) dst := newRegistryOfRepositories() dstServer := httptest.NewServer(dst.handler()) t.Cleanup(dstServer.Close) r := Registry{Address: strings.TrimPrefix(dstServer.URL, "http://"), HTTP: src.Client()} host := strings.TrimPrefix(src.URL, "http://") if _, err := r.MirrorImage(context.Background(), host+"/library/thing:latest", "hello-web/on-thing_base"); err != nil { t.Fatal(err) } return dst, r, host + "/library/thing@" + indexDigest, indexDigest, srcBlobs, src.Close } // An index whose platform manifests are not all held is not "already held": it is copied again, and the // copy puts back what is missing — what a sweep that stopped half-way, or ran while a build was copying, // leaves behind. func TestAnIndexMissingAPlatformIsCopiedAgain(t *testing.T) { dst, r, from, indexDigest, _, _ := heldUnderAModule(t) repository, _ := MirrorRepository(from) if _, err := r.MirrorBase(context.Background(), from, repository, ""); err != nil { t.Fatal(err) } // A sweep lets one platform go, between this build's copy and the next. var platform string for key := range dst.manifests { if strings.HasPrefix(key, repository+"@") && !strings.HasSuffix(key, indexDigest) { platform = key break } } delete(dst.manifests, platform) if _, err := r.MirrorBase(context.Background(), from, repository, ""); err != nil { t.Fatal(err) } if _, back := dst.manifests[platform]; !back { t.Fatalf("an index missing %s was taken as held", platform) } } // The same, for the module's former copy: a former copy missing a platform is not a source; upstream // is asked instead. func TestAFormerCopyMissingAPlatformIsNotTheSource(t *testing.T) { dst, r, from, indexDigest, _, _ := heldUnderAModule(t) for key := range dst.manifests { if strings.HasPrefix(key, "hello-web/on-thing_base@") && !strings.HasSuffix(key, indexDigest) { delete(dst.manifests, key) break } } repository, _ := MirrorRepository(from) if _, err := r.MirrorBase(context.Background(), from, repository, "hello-web/on-thing_base"); err != nil { t.Fatal(err) } if dst.mounts != 0 { t.Fatalf("a former copy missing a platform was mounted from (%d mounts)", dst.mounts) } if n := countPrefix(dst.manifests, repository+"@"); n != 3 { t.Fatalf("%d manifests copied from upstream, want 3", n) } } // A registry that answers a mount with an upload's location (202) gets the blob moved as before. func TestAMountRefusedFallsBackToAnUpload(t *testing.T) { dst, r, from, _, srcBlobs, _ := heldUnderAModule(t) dst.noMount = true uploaded := dst.uploads repository, _ := MirrorRepository(from) reference, err := r.MirrorBase(context.Background(), from, repository, "hello-web/on-thing_base") if err != nil { t.Fatal(err) } if !strings.HasSuffix(reference, repository+"@"+strings.SplitN(from, "@", 2)[1]) { t.Fatalf("pinned as %s", reference) } if dst.mounts != 0 || dst.uploads-uploaded != len(srcBlobs) { t.Fatalf("%d mounts, %d uploads; want every blob uploaded from the former copy", dst.mounts, dst.uploads-uploaded) } for digest := range srcBlobs { if !dst.links[repository][digest] { t.Fatalf("blob %s is not linked into %s", digest, repository) } } } // A sweep and a build at once: while builds copy the base, platforms are let go of under them. Each // build that returns has the copy whole. func TestABuildCopyingWhileASweepLetsGoEndsWhole(t *testing.T) { dst, r, from, indexDigest, _, _ := heldUnderAModule(t) repository, _ := MirrorRepository(from) if _, err := r.MirrorBase(context.Background(), from, repository, ""); err != nil { t.Fatal(err) } done := make(chan struct{}) go func() { defer close(done) for i := 0; i < 50; i++ { dst.mu.Lock() for key := range dst.manifests { if strings.HasPrefix(key, repository+"@") && !strings.HasSuffix(key, indexDigest) { delete(dst.manifests, key) break } } dst.mu.Unlock() } }() for i := 0; i < 20; i++ { if _, err := r.MirrorBase(context.Background(), from, repository, ""); err != nil { t.Fatal(err) } } <-done // The sweep is over; the next build finds what it left and makes the copy whole. if _, err := r.MirrorBase(context.Background(), from, repository, ""); err != nil { t.Fatal(err) } if n := countPrefix(dst.manifests, repository+"@"); n != 3 { t.Fatalf("after the sweep and the builds, %d manifests in %s, want 3", n, repository) } }