Raise a process-form controller at genesis as the container it replaces (hq issue 223)

The controller's manifest now declares a Go bundle the host runs as a
process (novox/hq issue 213). Genesis cannot run that: the bundle is
fetched from the artifact store and compiled in a toolchain, and the mesh
makes both long after the controller. The builder, asked to build the
manifest at genesis, refuses for lack of the Go toolchain. So genesis
raises the controller as before, as a container, and the first push hands
it over to the process through `replaces` (issue 223, option b).

- Step 3 clones the controller at the commit with the carried builder's
  git and reads its manifest. In the image form (an older controller) it
  builds through the builder as before. In the process form it builds the
  repository's own Dockerfile and hands step 9 a manifest of its own
  shape: the process becomes a container with the id the process
  `replaces`, the image genesis built, host network, and every host path
  the process's env names mounted at that same path read-only. Secrets
  belong to the image's user (65534) until the process's account takes
  them over. `prepares` is dropped: the temporary controller from the
  same commit already migrated the stores, and a pinned image is
  nothing the controller can derive a step from.
- Steps 4 to 9 are unchanged: they take the manifest as they did.
- apply.ForTests lets the bootstrap's test apply a process.

The first composed declaration from the process manifest names
`mesh-controller.server`, which is what the host recorded for the genesis
container, so the first apply hands over and leaves one controller.

internal/bootstrap/build.go

@@ -6,6 +6,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
+	"path/filepath"
 	"strings"
 )
 
@@ -112,6 +114,43 @@ func BuildControlPlane(ctx context.Context, run Runner, builderTag string, sourc
 		return Built{}, nil
 	}
 
+	// **Which form the controller is in at that commit** (novox/hq issue 223). An image the module
+	// builds is the form genesis always raised, and the builder builds it as before. A process the
+	// module runs from a Go bundle cannot be built here — its toolchain is one the mesh makes later —
+	// so genesis builds the controller's own Dockerfile and raises it as the container that process
+	// replaces (genesis_form.go).
+	workspace, err := os.MkdirTemp("", "mesh-genesis-*")
+	if err != nil {
+		return Built{}, err
+	}
+	defer os.RemoveAll(workspace)
+	dir, commit, err := cloneAt(ctx, run, builderTag, source, workspace)
+	if err != nil {
+		return Built{}, fmt.Errorf("the control plane could not be fetched from %s at %s: %w",
+			source.Repository, shortRef(source.Ref), err)
+	}
+	raw, err := os.ReadFile(filepath.Join(dir, "module.json"))
+	if err != nil {
+		return Built{}, fmt.Errorf("%s at %s has no module manifest: %w", source.Repository, shortRef(source.Ref), err)
+	}
+	form, err := processFormOf(raw)
+	if err != nil {
+		return Built{}, err
+	}
+	if form.Found {
+		image, err := buildGenesisImage(ctx, run, dir)
+		if err != nil {
+			return Built{}, err
+		}
+		manifest, err := genesisForm(raw, form, image)
+		if err != nil {
+			return Built{}, err
+		}
+		say(fmt.Sprintf("  built %s from %s, as the container its process %s replaces (%s)",
+			ControlPlaneModule, shortRef(commit), form.Process, form.Replaces))
+		return Built{Module: ControlPlaneModule, Commit: commit, Image: image, Manifest: manifest}, nil
+	}
+
 	out, err := run(ctx, "docker", args...)
 	if err != nil {
 		return Built{}, fmt.Errorf("the control plane could not be built from %s at %s: %w",

internal/bootstrap/build_test.go

@@ -49,10 +49,10 @@ func TestARepositoryAndACommitIsEnough(t *testing.T) {
 func TestTheBuildHandsOverTheManifestTheMeshWillHold(t *testing.T) {
 	manifest := `{"module":"mesh-controller","version":"1","resources":[` +
 		`{"id":"server","type":"container","name":"mesh-controller","image":"` + builtImage + `"}]}`
-	runtime := &asked{answer: func(string, []string) (string, error) {
+	runtime := &asked{answer: aRepository(t, imageFormManifest, func(string, []string) (string, error) {
 		return `{"module":"mesh-controller","commit":"a1b2c3d4","manifest":` + manifest +
 			`,"made":[{"name":"server","kind":"image","reference":"` + builtImage + `"}]}` + "\n", nil
-	}}
+	})}
 	built, err := BuildControlPlane(context.Background(), runtime.run, "mesh-builder:test",
 		Source{Repository: "https://example.invalid/mesh-controller.git", Ref: "a1b2c3d4"}, false, func(string) {})
 	if err != nil {
@@ -69,10 +69,10 @@ func TestTheBuildHandsOverTheManifestTheMeshWillHold(t *testing.T) {
 // A result without a manifest is a build the installer cannot finish, and it is refused beside the
 // builder that said it rather than at step 9 with a message about a missing file.
 func TestABuildReportingNoManifestIsRefused(t *testing.T) {
-	runtime := &asked{answer: func(string, []string) (string, error) {
+	runtime := &asked{answer: aRepository(t, imageFormManifest, func(string, []string) (string, error) {
 		return `{"module":"mesh-controller","commit":"a1b2c3d4",` +
 			`"made":[{"name":"server","kind":"image","reference":"` + builtImage + `"}]}`, nil
-	}}
+	})}
 	_, err := BuildControlPlane(context.Background(), runtime.run, "mesh-builder:test",
 		Source{Repository: "https://example.invalid/mesh-controller.git", Ref: "a1b2c3d4"}, false, func(string) {})
 	if err == nil || !strings.Contains(err.Error(), "no manifest") {
@@ -82,11 +82,11 @@ func TestABuildReportingNoManifestIsRefused(t *testing.T) {
 
 // And a manifest that does not name the image the build produced describes some other build.
 func TestABuildWhoseManifestNamesAnotherImageIsRefused(t *testing.T) {
-	runtime := &asked{answer: func(string, []string) (string, error) {
+	runtime := &asked{answer: aRepository(t, imageFormManifest, func(string, []string) (string, error) {
 		return `{"module":"mesh-controller","commit":"a1b2c3d4","manifest":{"module":"mesh-controller",` +
 			`"resources":[{"id":"server","type":"container","image":"sha256:` + strings.Repeat("9", 64) + `"}]},` +
 			`"made":[{"name":"server","kind":"image","reference":"` + builtImage + `"}]}`, nil
-	}}
+	})}
 	_, err := BuildControlPlane(context.Background(), runtime.run, "mesh-builder:test",
 		Source{Repository: "https://example.invalid/mesh-controller.git", Ref: "a1b2c3d4"}, false, func(string) {})
 	if err == nil || !strings.Contains(err.Error(), "does not name that image") {

internal/bootstrap/genesis_form.go

@@ -0,0 +1,205 @@
+package bootstrap
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// The controller as a process, raised at genesis as a container (novox/hq issue 213, issue 223).
+//
+// **The mesh runs the controller as a Go bundle the host starts as a process; genesis cannot.** A
+// process's bundle is fetched from the mesh's artifact store, which genesis raises long after the
+// controller, and a Go bundle is compiled in a toolchain the mesh builds later still. So genesis
+// pivots to the controller as it always has — an image it built from the controller's own
+// Dockerfile, run as a container the temporary controller composes — and the first time the mesh
+// builds the controller from its repository, the controller's own declaration is the process, which
+// names that container under `replaces`, and the host hands over: the process is started, seen up,
+// and only then is the container removed (mesh-host `replaces`, issue 213).
+//
+// **The container is genesis's shape, not the manifest's.** The manifest declares only the process.
+// From it genesis takes what the process is given — its environment, which is host paths and words —
+// and the id the process replaces; the container around it is written here: the image genesis built,
+// the host's network, and every host path the environment names mounted at the same path read-only.
+// It runs as the image's own unprivileged user (65534), so the secrets belong to that number until
+// the process's account takes them over — the image is FROM scratch and knows no account by name. Its
+// id is the one the process replaces, so the first declaration the mesh composes for this machine
+// hands this container over rather than leaving two controllers running.
+
+// genesisUser is who the genesis container runs as — the image's own USER — and who its secrets
+// belong to until the process takes them over: the image has no passwd to look an account up in.
+const genesisUser = "65534:65534"
+
+// ProcessForm is the controller's process, as its manifest declares it, and the container id that
+// process replaces. Found is false for a manifest in the image form — an older controller — which
+// genesis installs as it always did.
+type ProcessForm struct {
+	Found    bool
+	Process  string // the process resource's id
+	Replaces string // the id of the container genesis raises in its place
+}
+
+// processFormOf finds the controller's process in its manifest: a process resource running a bundle
+// the module builds, saying which one resource it replaces.
+func processFormOf(manifest []byte) (ProcessForm, error) {
+	var m struct {
+		Build *struct {
+			Artifacts []struct {
+				Name string `json:"name"`
+				Kind string `json:"kind"`
+			} `json:"artifacts"`
+		} `json:"build"`
+		Resources []map[string]any `json:"resources"`
+	}
+	if err := json.Unmarshal(manifest, &m); err != nil {
+		return ProcessForm{}, fmt.Errorf("the %s module's manifest is not readable: %w", ControlPlaneModule, err)
+	}
+	kinds := map[string]string{}
+	if m.Build != nil {
+		for _, a := range m.Build.Artifacts {
+			kinds[a.Name] = a.Kind
+		}
+	}
+	for _, kind := range kinds {
+		if kind == "image" {
+			return ProcessForm{}, nil // the image form: the builder builds it, as before
+		}
+	}
+	var found []ProcessForm
+	for _, r := range m.Resources {
+		if r["type"] != "process" || kinds[fmt.Sprint(r["artifact"])] != "bundle" {
+			continue
+		}
+		if once, _ := r["run-once"].(bool); once {
+			continue
+		}
+		replaces, _ := r["replaces"].([]any)
+		if len(replaces) != 1 {
+			return ProcessForm{}, fmt.Errorf(
+				"the %s module runs as the process %v and says it replaces %v. Genesis raises the "+
+					"controller as a container that process takes over, so the process names exactly "+
+					"one resource it replaces — the id genesis gives the container",
+				ControlPlaneModule, r["id"], r["replaces"])
+		}
+		found = append(found, ProcessForm{Found: true, Process: fmt.Sprint(r["id"]),
+			Replaces: fmt.Sprint(replaces[0])})
+	}
+	if len(found) != 1 {
+		return ProcessForm{}, fmt.Errorf(
+			"the %s module builds no image and runs %d process(es) of its own; genesis raises one "+
+				"controller, from the process its manifest declares", ControlPlaneModule, len(found))
+	}
+	return found[0], nil
+}
+
+// genesisForm is the manifest genesis registers: the controller's own manifest, its process
+// replaced by the container genesis runs in its place, under the id the process replaces, running
+// the image genesis built. Resolved as a build would resolve it — no build section, the image named
+// — because that is what the temporary controller is handed.
+func genesisForm(manifest []byte, form ProcessForm, image string) ([]byte, error) {
+	var m map[string]any
+	if err := json.Unmarshal(manifest, &m); err != nil {
+		return nil, err
+	}
+	resources, _ := m["resources"].([]any)
+	var out []any
+	for _, raw := range resources {
+		r, _ := raw.(map[string]any)
+		if r == nil || r["id"] != form.Process {
+			out = append(out, raw)
+			continue
+		}
+		env, _ := r["env"].(map[string]any)
+		var volumes []any
+		for _, key := range sortedAnyKeys(env) {
+			value := fmt.Sprint(env[key])
+			if strings.HasPrefix(value, "/") || strings.HasPrefix(value, "${dir:") {
+				volumes = append(volumes, value+":"+value+":ro")
+			}
+		}
+		container := map[string]any{
+			"id": form.Replaces, "type": "container", "name": ControlPlaneModule,
+			"image": image, "network": "host", "args": []any{"serve"},
+		}
+		if len(env) > 0 {
+			container["env"] = env
+		}
+		if len(volumes) > 0 {
+			container["volumes"] = volumes
+		}
+		out = append(out, container)
+	}
+	m["resources"] = out
+	// What the container reads must be readable by who it runs as. The process's account owns them
+	// once the process takes over, and the host gives them to it in the same apply.
+	m["secrets-owner"] = genesisUser
+	// **Nothing to prepare at genesis.** The temporary controller — the same commit — migrated the
+	// stores when the foundation raised it, and a preparation step is derived from a resource running
+	// an artifact the module built, which a pinned image is not: the controller refuses `prepares`
+	// with nothing to run it in. The process prepares the stores itself when it takes over.
+	delete(m, "prepares")
+	delete(m, "build")
+	var b bytes.Buffer
+	enc := json.NewEncoder(&b)
+	enc.SetEscapeHTML(false)
+	if err := enc.Encode(m); err != nil {
+		return nil, err
+	}
+	return bytes.TrimSpace(b.Bytes()), nil
+}
+
+func sortedAnyKeys(m map[string]any) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+// cloneAt fetches the controller's repository at the commit genesis builds, into a directory of
+// this machine's, with the carried builder's git — the machine is not assumed to have one. Returns
+// the module's directory and the commit that was checked out.
+func cloneAt(ctx context.Context, run Runner, builderTag string, source Source, into string) (string, string, error) {
+	git := func(args ...string) (string, error) {
+		return run(ctx, "docker", append([]string{"run", "--rm", "-v", into + ":/ws",
+			"--entrypoint", "git", builderTag}, args...)...)
+	}
+	if _, err := git("clone", "--quiet", source.Repository, "/ws/src"); err != nil {
+		return "", "", fmt.Errorf("cloning %s: %w", source.Repository, err)
+	}
+	if _, err := git("-C", "/ws/src", "checkout", "--quiet", "--detach", source.Ref); err != nil {
+		return "", "", fmt.Errorf("checking out %s: %w", shortRef(source.Ref), err)
+	}
+	commit, err := git("-C", "/ws/src", "rev-parse", "HEAD")
+	if err != nil {
+		return "", "", err
+	}
+	return filepath.Join(into, "src", source.Path), strings.TrimSpace(commit), nil
+}
+
+// buildGenesisImage builds the controller's image from its own Dockerfile (the one `make image`
+// uses), on this machine, and names it by the digest of its own configuration, as the builder did.
+func buildGenesisImage(ctx context.Context, run Runner, dir string) (string, error) {
+	if _, err := os.Stat(filepath.Join(dir, "Dockerfile")); err != nil {
+		return "", fmt.Errorf("the %s repository has no Dockerfile, so genesis has no image to raise "+
+			"the controller from: %w", ControlPlaneModule, err)
+	}
+	out, err := run(ctx, "docker", "build", "--quiet", dir)
+	if err != nil {
+		return "", fmt.Errorf("building the %s image: %w", ControlPlaneModule, err)
+	}
+	image := strings.TrimSpace(out)
+	if i := strings.LastIndex(image, "\n"); i >= 0 {
+		image = strings.TrimSpace(image[i+1:])
+	}
+	if !strings.HasPrefix(image, "sha256:") {
+		return "", fmt.Errorf("docker build said %q, which is not an image id", firstLine(out))
+	}
+	return image, nil
+}

internal/bootstrap/genesis_form_test.go

@@ -0,0 +1,182 @@
+package bootstrap
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// imageFormManifest is a controller from before issue 213: it builds an image and runs a container.
+const imageFormManifest = `{"module":"mesh-controller","version":"1","resources":[
+  {"id":"server","type":"container","name":"mesh-controller","network":"host","artifact":"server"}],
+  "build":{"artifacts":[{"name":"server","kind":"image","from":"Dockerfile"}]}}`
+
+// processFormManifest is the controller's manifest as novox/mesh-controller declares it after issue
+// 213: a Go bundle the host runs as a process, replacing the container it ran as.
+const processFormManifest = `{
+  "module": "mesh-controller", "version": "1", "slug": "control", "prepares": true,
+  "claims": [{"name": "mesh-controller", "scope": "mesh"}],
+  "accesses": [{"path": "/var/lib/mesh-broker-tls", "mode": "read"}],
+  "own-secrets": {"inventory": "${dir:mesh-state}/inventory", "bus": "${dir:mesh-state}/bus"},
+  "secrets-owner": "mesh-controller",
+  "tools": ["status"],
+  "resources": [
+    {"id": "account", "type": "user", "name": "mesh-controller", "shell": "/usr/bin/nologin", "home": "/var/lib/mesh-controller"},
+    {"id": "mesh-state", "type": "directory", "mode": "0700", "place": "mesh", "owner": "mesh-controller"},
+    {"id": "controller", "type": "process", "name": "mesh-controller", "artifact": "controller",
+     "run": ["./mesh-controller", "serve"], "user": "mesh-controller",
+     "env": {"MESH_BROKER_CERTIFICATE": "/var/lib/mesh-broker-tls/tls.crt",
+             "MESH_STORE_INVENTORY_FILE": "${dir:mesh-state}/inventory",
+             "MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}",
+             "MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus"},
+     "replaces": ["server"]}
+  ],
+  "build": {"artifacts": [{"name": "controller", "kind": "bundle", "language": "go", "system": "arch",
+    "from": "cmd/mesh-controller", "binary": "mesh-controller"}]}
+}`
+
+// aRepository answers the carried builder's git as a clone of a repository holding this manifest and
+// a Dockerfile, and hands every other command to then.
+func aRepository(t *testing.T, manifest string, then func(string, []string) (string, error)) func(string, []string) (string, error) {
+	t.Helper()
+	return func(name string, args []string) (string, error) {
+		if name == "docker" && len(args) > 5 && args[0] == "run" && args[4] == "--entrypoint" && args[5] == "git" {
+			host := strings.TrimSuffix(args[3], ":/ws")
+			joined := strings.Join(args, " ")
+			switch {
+			case strings.Contains(joined, " clone "):
+				src := filepath.Join(host, "src")
+				if err := os.MkdirAll(src, 0o755); err != nil {
+					return "", err
+				}
+				if err := os.WriteFile(filepath.Join(src, "module.json"), []byte(manifest), 0o644); err != nil {
+					return "", err
+				}
+				return "", os.WriteFile(filepath.Join(src, "Dockerfile"), []byte("FROM scratch\n"), 0o644)
+			case strings.Contains(joined, "rev-parse"):
+				return "a1b2c3d4e5f6\n", nil
+			}
+			return "", nil
+		}
+		return then(name, args)
+	}
+}
+
+// novox/hq issue 223: a controller declared as a process is raised at genesis as a container built
+// from its own Dockerfile, not by the builder — which has no Go toolchain at genesis and would refuse.
+func TestAProcessFormControllerIsBuiltFromItsDockerfileAndRaisedAsAContainer(t *testing.T) {
+	runtime := &asked{answer: aRepository(t, processFormManifest, func(name string, args []string) (string, error) {
+		if name == "docker" && args[0] == "build" {
+			return builtImage + "\n", nil
+		}
+		t.Fatalf("genesis ran %s %v; a process-form controller is built from its Dockerfile alone", name, args)
+		return "", nil
+	})}
+	built, err := BuildControlPlane(context.Background(), runtime.run, "mesh-builder:test",
+		Source{Repository: "https://example.invalid/mesh-controller.git", Ref: "a1b2c3d4"}, false, func(string) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if built.Image != builtImage || built.Commit != "a1b2c3d4e5f6" {
+		t.Errorf("built %q from %q", built.Image, built.Commit)
+	}
+	if runtime.ran("mesh-builder:test build") {
+		t.Error("the builder was asked to build a controller it cannot build at genesis")
+	}
+
+	var m map[string]any
+	if err := json.Unmarshal(built.Manifest, &m); err != nil {
+		t.Fatalf("the genesis manifest is not JSON: %v", err)
+	}
+	if _, has := m["prepares"]; has {
+		t.Error("the genesis manifest prepares its state; the temporary controller already did, and a pinned image is nothing the controller can derive a step from")
+	}
+	if _, has := m["build"]; has {
+		t.Error("the genesis manifest still says how it is built; it is handed over resolved")
+	}
+	var container map[string]any
+	for _, raw := range m["resources"].([]any) {
+		r := raw.(map[string]any)
+		if r["type"] == "process" {
+			t.Errorf("the genesis manifest still runs the process: %v", r)
+		}
+		if r["type"] == "container" {
+			container = r
+		}
+	}
+	if container == nil {
+		t.Fatal("the genesis manifest runs no container")
+	}
+	for key, want := range map[string]any{"id": "server", "name": "mesh-controller", "image": builtImage,
+		"network": "host"} {
+		if container[key] != want {
+			t.Errorf("the genesis container's %s is %v, not %v", key, container[key], want)
+		}
+	}
+	if _, has := container["user"]; has {
+		t.Error("the genesis container says a user; the host's container has no such field, the image's USER is who it runs as")
+	}
+	if m["secrets-owner"] != "65534:65534" {
+		t.Errorf("the secrets belong to %v, which the container cannot read as", m["secrets-owner"])
+	}
+	volumes, _ := json.Marshal(container["volumes"])
+	for _, want := range []string{"${dir:mesh-state}/inventory:${dir:mesh-state}/inventory:ro",
+		"/var/lib/mesh-broker-tls/tls.crt:/var/lib/mesh-broker-tls/tls.crt:ro"} {
+		if !strings.Contains(string(volumes), want) {
+			t.Errorf("the container does not mount %s: %s", want, volumes)
+		}
+	}
+	if strings.Contains(string(volumes), "seat:") {
+		t.Errorf("a word that is not a path was mounted: %s", volumes)
+	}
+
+	// And it is what step 9 installs: pinned, its container found, its stores delivered from its
+	// environment — the same path an image-form controller takes.
+	pinned, _, err := pinImage(built.Manifest, built.Image, "registry.internal:5000/mesh-controller@sha256:"+strings.Repeat("e", 64), ControlPlaneModule)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if id := controlPlaneResourceIn(pinned); id != "server" {
+		t.Errorf("step 9 finds the controller's container as %q", id)
+	}
+	wanted, err := secretsByVariableIn(pinned)
+	if err != nil {
+		t.Fatalf("step 9 cannot deliver the stores into the genesis container: %v", err)
+	}
+	if wanted["MESH_STORE_INVENTORY"] != "inventory" {
+		t.Errorf("step 9 delivers %v", wanted)
+	}
+}
+
+// An older controller — an image and a container — is still built by the builder, as before.
+func TestAnImageFormControllerIsStillBuiltByTheBuilder(t *testing.T) {
+	manifest := `{"module":"mesh-controller","version":"1","resources":[` +
+		`{"id":"server","type":"container","name":"mesh-controller","image":"` + builtImage + `"}]}`
+	runtime := &asked{answer: aRepository(t, imageFormManifest, func(name string, args []string) (string, error) {
+		if name == "docker" && args[0] == "build" {
+			t.Fatal("an image-form controller was built from its Dockerfile rather than by the builder")
+		}
+		return `{"module":"mesh-controller","commit":"a1b2c3d4","manifest":` + manifest +
+			`,"made":[{"name":"server","kind":"image","reference":"` + builtImage + `"}]}`, nil
+	})}
+	built, err := BuildControlPlane(context.Background(), runtime.run, "mesh-builder:test",
+		Source{Repository: "https://example.invalid/mesh-controller.git", Ref: "a1b2c3d4"}, false, func(string) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(built.Manifest) != manifest || !runtime.ran("mesh-builder:test build") {
+		t.Errorf("the image form did not go through the builder: %s", built.Manifest)
+	}
+}
+
+func TestAProcessFormNamingNoOneReplacementIsRefused(t *testing.T) {
+	for _, bad := range []string{`"replaces": []`, `"replaces": ["a", "b"]`} {
+		raw := strings.Replace(processFormManifest, `"replaces": ["server"]`, bad, 1)
+		if _, err := processFormOf([]byte(raw)); err == nil {
+			t.Errorf("a process saying %s was accepted; genesis would not know what to name its container", bad)
+		}
+	}
+}

internal/bootstrap/genesis_handover_test.go

@@ -0,0 +1,121 @@
+package bootstrap
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/apply"
+	"github.com/novox/mesh-host/internal/declaration"
+	"github.com/novox/mesh-host/internal/store"
+	"github.com/novox/mesh-host/internal/system"
+)
+
+// novox/hq issue 223: what genesis raises is what the controller's process takes over. The temporary
+// controller composes the genesis container, and the host records it as `<module>.<its id>`; the
+// first declaration the mesh composes from the controller's real manifest names that same id under
+// the process's `replaces` (the composer prefixes both alike — mesh-controller's
+// TestTheControllerIsAProcessAndNoContainer). So the first apply hands over: the process is started,
+// seen up, and only then is the genesis container removed — one controller before, one after, never
+// none and never two left.
+func TestTheFirstApplyHandsTheGenesisContainerOverToTheProcess(t *testing.T) {
+	restore := apply.ForTests(t.TempDir(), t.TempDir())
+	defer restore()
+
+	form, err := processFormOf([]byte(processFormManifest))
+	if err != nil {
+		t.Fatal(err)
+	}
+	genesis, err := genesisForm([]byte(processFormManifest), form, builtImage)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// What the host records for the genesis container, as the composer names it.
+	recorded := ""
+	var m struct {
+		Resources []map[string]any `json:"resources"`
+	}
+	if err := json.Unmarshal(genesis, &m); err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range m.Resources {
+		if r["type"] == "container" {
+			recorded = ControlPlaneModule + "." + r["id"].(string)
+		}
+	}
+	known := store.State{Resources: []store.Applied{{Origin: store.OriginDeclared, ID: recorded,
+		Type: "container", Target: ControlPlaneModule}}}
+
+	// The controller's first composed declaration: its process, replacing what the manifest names.
+	body, digest := aBundle(t)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(body) }))
+	defer server.Close()
+	replaces, _ := json.Marshal([]string{ControlPlaneModule + "." + form.Replaces})
+	d, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"` + ControlPlaneModule + "." + form.Process +
+		`","type":"process","name":"mesh-controller","source":"` + server.URL + `/c.tgz","digest":"` + digest +
+		`","run":["./mesh-controller","serve"],"replaces":` + string(replaces) + `}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var commands []string
+	started, container := false, true
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		line := name + " " + strings.Join(args, " ")
+		commands = append(commands, line)
+		switch {
+		case strings.HasPrefix(line, "systemctl restart mesh-controller.service"):
+			started = true
+		case strings.HasPrefix(line, "systemctl show mesh-controller.service") && started:
+			return "ActiveState=active\nSubState=running\nMainPID=7\nNRestarts=0\n", nil
+		case strings.HasPrefix(line, "docker rm -f mesh-controller"):
+			if !started {
+				t.Error("the genesis container was removed before the process was started")
+			}
+			container = false
+		case strings.HasPrefix(line, "docker container inspect") && !container:
+			return "", errors.New("no such container")
+		}
+		return "", nil
+	}
+	sys, err := system.For("arch")
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, after, err := apply.Apply(context.Background(), sys, d, known, store.OriginDeclared, run, nil, nil)
+	if err != nil {
+		t.Fatalf("the first apply did not hand over: %v\n%s", err, strings.Join(commands, "\n"))
+	}
+	if container {
+		t.Fatalf("the genesis container is still running beside the process — two controllers:\n%s",
+			strings.Join(commands, "\n"))
+	}
+	if _, still := after.Find(recorded); still {
+		t.Error("the host still records the genesis container")
+	}
+}
+
+func aBundle(t *testing.T) ([]byte, string) {
+	t.Helper()
+	var raw bytes.Buffer
+	zipped := gzip.NewWriter(&raw)
+	w := tar.NewWriter(zipped)
+	content := "#!/bin/sh\n"
+	if err := w.WriteHeader(&tar.Header{Name: "mesh-controller", Mode: 0o755, Size: int64(len(content)), Typeflag: tar.TypeReg}); err != nil {
+		t.Fatal(err)
+	}
+	_, _ = w.Write([]byte(content))
+	_ = w.Close()
+	_ = zipped.Close()
+	sum := sha256.Sum256(raw.Bytes())
+	return raw.Bytes(), "sha256:" + hex.EncodeToString(sum[:])
+}