A module may mirror an image it did not write

A module usually runs software somebody else built: a database module
ships configuration and a provisioner and does not build a database. It
could name the upstream reference directly, and then every machine needs
a route to a public registry and the reference is a tag somebody else can
move — which is what pinning exists to prevent.

So an artifact may be `upstream`: pulled by the reference the module
names, pushed into the mesh's own registry, and pinned by the digest that
registry assigns. This is what the bootstrap already does by hand; it is
now something a module can say.

Refused: an upstream reference with no tag or digest, because what gets
mirrored would be whatever `latest` means today and a module pinned to
that is not pinned. And the rule that a build reads only its own
repository does not apply to it — applying it anyway refused every
reference with a registry host in it, which the test caught.

Written by trying to write a real postgres module and finding it could
not be said. It can now: two directories, two containers pinned by
digest, a superuser password sealed to the machine, and the grants
manifest — six resources from one assignment, all accepted by the host's
own parser.

That exercise also found my manifest wrong rather than the host: a
container declared `restart-on`, which is a service field, and the host
refused it by name. It is right to. A container whose own definition
changes is recreated, and a file it mounts is read by the process inside,
which is that image's business.

internal/builder/builder.go

@@ -128,6 +128,19 @@ func one(ctx context.Context, run Runner, publish Publisher,
 	module, tree, commit string, a catalogue.Artifact) (catalogue.Built, error) {
 
 	switch a.Kind {
+	case catalogue.ArtifactUpstream:
+		// Mirrored, not built. Pulled by the reference the module names and pushed under a name
+		// of the mesh's own, so what a machine fetches is pinned by a digest this registry
+		// assigned rather than by a tag somebody else can move.
+		if _, err := run(ctx, tree, "docker", "pull", a.From); err != nil {
+			return catalogue.Built{}, fmt.Errorf("%s: cannot fetch %s: %w", module, a.From, err)
+		}
+		reference, err := publish.PublishImage(ctx, a.From, module+"/"+a.Name)
+		if err != nil {
+			return catalogue.Built{}, err
+		}
+		return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
+
 	case catalogue.ArtifactImage:
 		// Tagged by commit rather than by version, because a version is what a person calls a
 		// release and a commit is what was actually built. The mesh pins the digest anyway; this

internal/builder/builder_test.go

@@ -226,3 +226,64 @@ func TestABuildThatCannotPushFails(t *testing.T) {
 		t.Fatal("a build that could publish nothing reported success")
 	}
 }
+
+func TestAnUpstreamImageIsMirroredRatherThanBuilt(t *testing.T) {
+	// A module usually runs software it did not write. Naming the upstream reference directly
+	// would need every machine to reach a public registry, and would pin to a tag somebody else
+	// can move.
+	const mirrors = `{"module":"postgres","version":"1",
+	  "build":{"artifacts":[{"name":"store","kind":"upstream","from":"postgres:17-alpine"}]},
+	  "resources":[{"id":"db","type":"container","name":"mesh-postgres","artifact":"store"}]}`
+
+	r, workspace := aRepository(t, mirrors, nil)
+	got, err := Build(context.Background(), r.run, r, "https://forge.invalid/postgres.git", "", workspace)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Pulled, not built.
+	var pulled, built bool
+	for _, line := range r.ran {
+		if strings.HasPrefix(line, "docker pull postgres:17-alpine") {
+			pulled = true
+		}
+		if strings.HasPrefix(line, "docker build") {
+			built = true
+		}
+	}
+	if !pulled {
+		t.Fatalf("the upstream image was not fetched: %v", r.ran)
+	}
+	if built {
+		t.Fatalf("something was built for an image that is mirrored: %v", r.ran)
+	}
+	// And the resource names what this registry serves, pinned by the digest it assigned.
+	image, _ := got.Manifest.Resources[0]["image"].(string)
+	if !strings.Contains(image, "@sha256:") {
+		t.Fatalf("the mirrored image is not pinned by digest: %q", image)
+	}
+	if strings.Contains(image, "17-alpine") {
+		t.Fatalf("the resource still names the upstream tag: %q", image)
+	}
+}
+
+func TestAnUpstreamImageWithNoTagIsRefused(t *testing.T) {
+	// What gets mirrored would be whatever `latest` means today, and a module pinned to that is
+	// not pinned.
+	_, err := catalogue.ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[
+		{"name":"x","kind":"upstream","from":"postgres"}]}}`))
+	if err == nil {
+		t.Fatal("an untagged upstream reference was accepted")
+	}
+	if !strings.Contains(err.Error(), "no tag or digest") {
+		t.Fatalf("unhelpful refusal: %v", err)
+	}
+}
+
+func TestAnUpstreamReferenceIsNotAPathInTheRepository(t *testing.T) {
+	// The rule that a build reads only its own repository must not refuse every reference with a
+	// registry host in it.
+	if _, err := catalogue.ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[
+		{"name":"x","kind":"upstream","from":"registry.example/library/postgres:17"}]}}`)); err != nil {
+		t.Fatalf("a perfectly ordinary upstream reference was refused: %v", err)
+	}
+}

internal/catalogue/build.go

@@ -86,14 +86,14 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) {
 		}
 		delete(filled, "artifact")
 		switch artifact.Kind {
-		case ArtifactImage:
+		case ArtifactImage, ArtifactUpstream:
 			filled["image"] = artifact.Reference
 		case ArtifactArchive:
 			filled["source"] = artifact.Reference
 			filled["digest"] = artifact.Digest
 		default:
-			return Manifest{}, fmt.Errorf("%s: %q is a %q, and an artifact is %q or %q",
-				m.Module, named, artifact.Kind, ArtifactImage, ArtifactArchive)
+			return Manifest{}, fmt.Errorf("%s: %q is a %q, and an artifact is %q, %q or %q",
+				m.Module, named, artifact.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream)
 		}
 		out.Resources = append(out.Resources, filled)
 	}
@@ -118,21 +118,34 @@ func (b *Build) problems(module string) []string {
 				module, a.Name))
 		}
 		seen[a.Name] = true
-		if a.Kind != ArtifactImage && a.Kind != ArtifactArchive {
-			problems = append(problems, fmt.Sprintf("%s: %q is a %q, and an artifact is %q or %q",
-				module, a.Name, a.Kind, ArtifactImage, ArtifactArchive))
+		switch a.Kind {
+		case ArtifactImage, ArtifactArchive, ArtifactUpstream:
+		default:
+			problems = append(problems, fmt.Sprintf(
+				"%s: %q is a %q, and an artifact is %q, %q or %q",
+				module, a.Name, a.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream))
 		}
 		if a.From == "" {
 			problems = append(problems, fmt.Sprintf(
 				"%s: %q says nothing about what it is built from", module, a.Name))
 		}
-		if strings.HasPrefix(a.From, "/") || strings.Contains(a.From, "..") {
+		// An upstream image is named, not read from the repository, so the path rule does not
+		// apply to it — and applying it anyway would refuse every reference with a registry host
+		// in it.
+		if a.Kind != ArtifactUpstream &&
+			(strings.HasPrefix(a.From, "/") || strings.Contains(a.From, "..")) {
 			// A build reads its own repository and nothing else. A path leaving it would make
 			// what gets built depend on whatever happens to be on the machine building it.
 			problems = append(problems, fmt.Sprintf(
 				"%s: %q is built from %q, which is outside its own repository",
 				module, a.Name, a.From))
 		}
+		if a.Kind == ArtifactUpstream && !strings.Contains(a.From, ":") {
+			// Without a tag or digest, what gets mirrored is whatever `latest` means today, and
+			// a module pinned to that is not pinned.
+			problems = append(problems, fmt.Sprintf(
+				"%s: %q mirrors %q, which names no tag or digest", module, a.Name, a.From))
+		}
 	}
 	return problems
 }

internal/catalogue/manifest.go

@@ -244,8 +244,22 @@ type Artifact struct {
 
 // Kinds an artifact may be.
 const (
-	ArtifactImage   = "image"
+	// ArtifactImage is built from a Dockerfile in this repository.
+	ArtifactImage = "image"
+	// ArtifactArchive is a directory in this repository, packed.
 	ArtifactArchive = "archive"
+	// ArtifactUpstream is an image somebody else built, mirrored into the mesh's own registry and
+	// pinned by the digest it lands with.
+	//
+	// **Because a module usually runs software it did not write.** A database module ships
+	// configuration and a provisioner and does not build a database. It could name the upstream
+	// reference directly, and then every machine needs a route to a public registry and the
+	// reference is a tag somebody else can move — which is what pinning exists to prevent
+	// (novox/hq ADR 0006).
+	//
+	// Mirroring is what the bootstrap already does by hand: the lab stocks upstream images into
+	// the registry a first node pulls from. This makes that a thing a module can say.
+	ArtifactUpstream = "upstream"
 )
 
 // NeedID is the resource identity of the file a module's own secret lands in.