From 15fd70e3ceb933f70ae57b367527c4cf71157e0e Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 30 Aug 2026 18:28:05 +0200 Subject: [PATCH] A module may mirror an image it did not write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A module usually runs software somebody else built: a database module ships configuration and a provisioner and does not build a database. It could name the upstream reference directly, and then every machine needs a route to a public registry and the reference is a tag somebody else can move — which is what pinning exists to prevent. So an artifact may be `upstream`: pulled by the reference the module names, pushed into the mesh's own registry, and pinned by the digest that registry assigns. This is what the bootstrap already does by hand; it is now something a module can say. Refused: an upstream reference with no tag or digest, because what gets mirrored would be whatever `latest` means today and a module pinned to that is not pinned. And the rule that a build reads only its own repository does not apply to it — applying it anyway refused every reference with a registry host in it, which the test caught. Written by trying to write a real postgres module and finding it could not be said. It can now: two directories, two containers pinned by digest, a superuser password sealed to the machine, and the grants manifest — six resources from one assignment, all accepted by the host's own parser. That exercise also found my manifest wrong rather than the host: a container declared `restart-on`, which is a service field, and the host refused it by name. It is right to. A container whose own definition changes is recreated, and a file it mounts is read by the process inside, which is that image's business. --- internal/builder/builder.go | 13 +++++++ internal/builder/builder_test.go | 61 ++++++++++++++++++++++++++++++++ internal/catalogue/build.go | 27 ++++++++++---- internal/catalogue/manifest.go | 16 ++++++++- 4 files changed, 109 insertions(+), 8 deletions(-) diff --git a/internal/builder/builder.go b/internal/builder/builder.go index a558711..7574280 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -128,6 +128,19 @@ func one(ctx context.Context, run Runner, publish Publisher, module, tree, commit string, a catalogue.Artifact) (catalogue.Built, error) { switch a.Kind { + case catalogue.ArtifactUpstream: + // Mirrored, not built. Pulled by the reference the module names and pushed under a name + // of the mesh's own, so what a machine fetches is pinned by a digest this registry + // assigned rather than by a tag somebody else can move. + if _, err := run(ctx, tree, "docker", "pull", a.From); err != nil { + return catalogue.Built{}, fmt.Errorf("%s: cannot fetch %s: %w", module, a.From, err) + } + reference, err := publish.PublishImage(ctx, a.From, module+"/"+a.Name) + if err != nil { + return catalogue.Built{}, err + } + return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil + case catalogue.ArtifactImage: // Tagged by commit rather than by version, because a version is what a person calls a // release and a commit is what was actually built. The mesh pins the digest anyway; this diff --git a/internal/builder/builder_test.go b/internal/builder/builder_test.go index 1380afa..0719b1f 100644 --- a/internal/builder/builder_test.go +++ b/internal/builder/builder_test.go @@ -226,3 +226,64 @@ func TestABuildThatCannotPushFails(t *testing.T) { t.Fatal("a build that could publish nothing reported success") } } + +func TestAnUpstreamImageIsMirroredRatherThanBuilt(t *testing.T) { + // A module usually runs software it did not write. Naming the upstream reference directly + // would need every machine to reach a public registry, and would pin to a tag somebody else + // can move. + const mirrors = `{"module":"postgres","version":"1", + "build":{"artifacts":[{"name":"store","kind":"upstream","from":"postgres:17-alpine"}]}, + "resources":[{"id":"db","type":"container","name":"mesh-postgres","artifact":"store"}]}` + + r, workspace := aRepository(t, mirrors, nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/postgres.git", "", workspace) + if err != nil { + t.Fatal(err) + } + // Pulled, not built. + var pulled, built bool + for _, line := range r.ran { + if strings.HasPrefix(line, "docker pull postgres:17-alpine") { + pulled = true + } + if strings.HasPrefix(line, "docker build") { + built = true + } + } + if !pulled { + t.Fatalf("the upstream image was not fetched: %v", r.ran) + } + if built { + t.Fatalf("something was built for an image that is mirrored: %v", r.ran) + } + // And the resource names what this registry serves, pinned by the digest it assigned. + image, _ := got.Manifest.Resources[0]["image"].(string) + if !strings.Contains(image, "@sha256:") { + t.Fatalf("the mirrored image is not pinned by digest: %q", image) + } + if strings.Contains(image, "17-alpine") { + t.Fatalf("the resource still names the upstream tag: %q", image) + } +} + +func TestAnUpstreamImageWithNoTagIsRefused(t *testing.T) { + // What gets mirrored would be whatever `latest` means today, and a module pinned to that is + // not pinned. + _, err := catalogue.ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[ + {"name":"x","kind":"upstream","from":"postgres"}]}}`)) + if err == nil { + t.Fatal("an untagged upstream reference was accepted") + } + if !strings.Contains(err.Error(), "no tag or digest") { + t.Fatalf("unhelpful refusal: %v", err) + } +} + +func TestAnUpstreamReferenceIsNotAPathInTheRepository(t *testing.T) { + // The rule that a build reads only its own repository must not refuse every reference with a + // registry host in it. + if _, err := catalogue.ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[ + {"name":"x","kind":"upstream","from":"registry.example/library/postgres:17"}]}}`)); err != nil { + t.Fatalf("a perfectly ordinary upstream reference was refused: %v", err) + } +} diff --git a/internal/catalogue/build.go b/internal/catalogue/build.go index 31ff2b0..361c65e 100644 --- a/internal/catalogue/build.go +++ b/internal/catalogue/build.go @@ -86,14 +86,14 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) { } delete(filled, "artifact") switch artifact.Kind { - case ArtifactImage: + case ArtifactImage, ArtifactUpstream: filled["image"] = artifact.Reference case ArtifactArchive: filled["source"] = artifact.Reference filled["digest"] = artifact.Digest default: - return Manifest{}, fmt.Errorf("%s: %q is a %q, and an artifact is %q or %q", - m.Module, named, artifact.Kind, ArtifactImage, ArtifactArchive) + return Manifest{}, fmt.Errorf("%s: %q is a %q, and an artifact is %q, %q or %q", + m.Module, named, artifact.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream) } out.Resources = append(out.Resources, filled) } @@ -118,21 +118,34 @@ func (b *Build) problems(module string) []string { module, a.Name)) } seen[a.Name] = true - if a.Kind != ArtifactImage && a.Kind != ArtifactArchive { - problems = append(problems, fmt.Sprintf("%s: %q is a %q, and an artifact is %q or %q", - module, a.Name, a.Kind, ArtifactImage, ArtifactArchive)) + switch a.Kind { + case ArtifactImage, ArtifactArchive, ArtifactUpstream: + default: + problems = append(problems, fmt.Sprintf( + "%s: %q is a %q, and an artifact is %q, %q or %q", + module, a.Name, a.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream)) } if a.From == "" { problems = append(problems, fmt.Sprintf( "%s: %q says nothing about what it is built from", module, a.Name)) } - if strings.HasPrefix(a.From, "/") || strings.Contains(a.From, "..") { + // An upstream image is named, not read from the repository, so the path rule does not + // apply to it — and applying it anyway would refuse every reference with a registry host + // in it. + if a.Kind != ArtifactUpstream && + (strings.HasPrefix(a.From, "/") || strings.Contains(a.From, "..")) { // A build reads its own repository and nothing else. A path leaving it would make // what gets built depend on whatever happens to be on the machine building it. problems = append(problems, fmt.Sprintf( "%s: %q is built from %q, which is outside its own repository", module, a.Name, a.From)) } + if a.Kind == ArtifactUpstream && !strings.Contains(a.From, ":") { + // Without a tag or digest, what gets mirrored is whatever `latest` means today, and + // a module pinned to that is not pinned. + problems = append(problems, fmt.Sprintf( + "%s: %q mirrors %q, which names no tag or digest", module, a.Name, a.From)) + } } return problems } diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 482c4b4..644f825 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -244,8 +244,22 @@ type Artifact struct { // Kinds an artifact may be. const ( - ArtifactImage = "image" + // ArtifactImage is built from a Dockerfile in this repository. + ArtifactImage = "image" + // ArtifactArchive is a directory in this repository, packed. ArtifactArchive = "archive" + // ArtifactUpstream is an image somebody else built, mirrored into the mesh's own registry and + // pinned by the digest it lands with. + // + // **Because a module usually runs software it did not write.** A database module ships + // configuration and a provisioner and does not build a database. It could name the upstream + // reference directly, and then every machine needs a route to a public registry and the + // reference is a tag somebody else can move — which is what pinning exists to prevent + // (novox/hq ADR 0006). + // + // Mirroring is what the bootstrap already does by hand: the lab stocks upstream images into + // the registry a first node pulls from. This makes that a thing a module can say. + ArtifactUpstream = "upstream" ) // NeedID is the resource identity of the file a module's own secret lands in.