From 0a88dc1f9d2e760310d0b40615cfdfc9ea7f69ee Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 11 Sep 2026 00:03:14 +0200 Subject: [PATCH] bootstrap: follow the mount from the variable to the secret MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The catalogue's mesh-control manifest landed while this was being written, and it does what the ordinary case does: it keeps its secrets under /var/lib/mesh and mounts them into the container at /run/secrets, so MESH_STORE_INVENTORY_FILE names a path that no own-secret writes. Matching on the path alone found nothing and would have refused a correct manifest. So the lookup follows the volumes. It also reads the other shape the manifest uses — `VAR=${secret:name}` inside the environment file a container reads — which is how a value that is not a path gets in at all, and which is where the broker's two credentials live. That generalises what is delivered: every variable the module fills from a secret is looked up in the substrate's control plane. What the substrate names is accepted through `secret accept`; what it does not is left for the mesh to generate, and said so. A store connection the substrate does not name stays an error — a control plane that cannot open a context is not one. Checked against the real manifest (mesh-catalog feat/control-plane-module): five variables resolve, the placeholder pins in one place, and the container it waits for is `mesh-control`. Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- internal/bootstrap/control.go | 181 ++++++++++++++++++++--------- internal/bootstrap/control_test.go | 118 ++++++++++++++----- 2 files changed, 218 insertions(+), 81 deletions(-) diff --git a/internal/bootstrap/control.go b/internal/bootstrap/control.go index c406b53..6f6d937 100644 --- a/internal/bootstrap/control.go +++ b/internal/bootstrap/control.go @@ -218,26 +218,32 @@ func controlPlaneResourceIn(manifest []byte) string { // deliverStores carries the substrate's own database connections into the module. // // The pairing is read from the manifest rather than assumed, so that whatever the catalogue calls -// these secrets is what is delivered: a container asking for `MESH_STORE_INVENTORY_FILE` names a -// path, and the module's own-secret that writes that path is the secret to accept the connection -// as. That is one lookup and it cannot get the wrong secret — the alternative, guessing that the -// secret is called `inventory`, would seal a connection string under a name nothing reads and -// leave the mesh to invent random bytes for the one that is. +// these secrets is what is delivered. The installer does not guess that the secret holding the +// inventory connection is called `inventory`; it follows the manifest from the variable to the +// secret, and a manifest whose two ends do not meet is refused rather than half-delivered. +// +// **What is delivered is what the substrate already has, and only that.** The mesh generates an +// own-secret nobody supplied, which is right for something coming into existence and wrong for +// something that already exists. So every variable the module fills from a secret is looked up in +// the substrate's control plane: what it names is accepted, what it does not is left for the mesh +// to make. A store connection missing from the substrate is the one exception and is an error — +// a control plane that cannot open a context is not a control plane. func deliverStores(ctx context.Context, o Options, control controlPlane, manifest []byte, substrate *declaration.Declaration, say func(string)) ([]string, error) { - wanted, err := storeSecretsIn(manifest) + wanted, err := secretsByVariableIn(manifest) if err != nil { return nil, err } - if len(wanted) == 0 { + if !anyStoreIn(wanted) { return nil, fmt.Errorf( - "the %s module's manifest asks for no store connections. A control plane reaches each "+ - "context through its own credential (novox/hq ADR 0008), so a manifest naming none "+ - "describes a control plane that can open nothing.\n"+ + "the %s module's manifest fills no %s… variable from a secret. A control plane reaches "+ + "each context through its own credential (novox/hq ADR 0008), so a manifest naming "+ + "none describes a control plane that can open nothing.\n"+ "The shape this installer delivers into is a file per context, named by an "+ - "own-secret, with %s%s in the container's environment pointing at it", - ControlPlaneModule, storeVariablePrefix, storeFileSuffix) + "own-secret, with %s%s pointing at it — directly, or at where that file is "+ + "mounted inside the container", + ControlPlaneModule, storeVariablePrefix, storeVariablePrefix, storeFileSuffix) } temporary, err := controlPlaneIn(substrate) @@ -246,24 +252,29 @@ func deliverStores(ctx context.Context, o Options, control controlPlane, manifes } var delivered []string - for _, context := range sortedKeys(wanted) { - secret := wanted[context] - connection := temporary.Env[storeVariablePrefix+context] - if strings.TrimSpace(connection) == "" { - return delivered, fmt.Errorf( - "the %s module wants the %s store's connection and the bundle this installer "+ - "produced does not name one: its control plane has no %s.\n"+ - "These connections are the substrate's, created at genesis — the mesh cannot "+ - "invent them and the installer will not guess at one", - ControlPlaneModule, strings.ToLower(context), storeVariablePrefix+context) + for _, variable := range sortedKeys(wanted) { + secret := wanted[variable] + value := strings.TrimSpace(temporary.Env[variable]) + if value == "" { + if strings.HasPrefix(variable, storeVariablePrefix) { + return delivered, fmt.Errorf( + "the %s module wants %s and the bundle this installer produced does not name "+ + "one.\n"+ + "That connection is the substrate's, created at genesis — the mesh cannot "+ + "invent it and the installer will not guess at one", + ControlPlaneModule, variable) + } + // Not something the substrate made. The mesh generates its own, which is exactly what + // an own-secret is for; said so that nothing about the delivery is silent. + say(" the mesh will make " + secret + " — the substrate names no " + variable) + continue } // Into the container as a file, because `secret accept` reads a file or a prompt and the // installer has neither a terminal to be prompted at nor a way to write to a command's // standard input through the runner every applier in this repository shares. - at := "/accepting-" + strings.ToLower(context) - if err := control.carrying(ctx, "mesh-store-"+strings.ToLower(context), - []byte(connection), at); err != nil { + at := "/accepting-" + secret + if err := control.carrying(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil { return delivered, err } if _, err := control.tell(ctx, "secret", "accept", o.Node, ControlPlaneModule, secret, @@ -271,60 +282,124 @@ func deliverStores(ctx context.Context, o Options, control controlPlane, manifes return delivered, err } delivered = append(delivered, secret) - say(" accepted " + secret + " — the " + strings.ToLower(context) + - " store, as the substrate made it") + say(" accepted " + secret + " — " + variable + ", as the substrate made it") } return delivered, nil } -// storeSecretsIn pairs each context with the secret its connection must be accepted as. +// secretsByVariableIn maps each environment variable the module fills from a secret to that +// secret's name. // -// Read out of the manifest twice over: the container's environment says which contexts are wanted -// and what file each expects, and the module's own-secrets say which secret writes which file. A -// pair that does not meet is refused rather than half-delivered. -func storeSecretsIn(manifest []byte) (map[string]string, error) { +// Two shapes, because the catalogue uses both: +// +// - `MESH_STORE__FILE` in the container's environment, naming a path the process reads. +// The path may be the own-secret's own path, or — more usually — where that file is mounted +// inside the container, in which case the volumes say which is which. Following the mount is +// not a nicety: a manifest that keeps its secrets under `/var/lib/mesh/…` and mounts them at +// `/run/secrets/…` is the ordinary case, and matching on the path alone would find nothing and +// refuse a correct manifest. +// - `VAR=${secret:name}` inside a file resource the container reads its environment from, which +// is how a value that is not a path gets in at all. +// +// A `…_FILE` variable whose file nothing writes is refused: the mesh would seal nothing there and +// the process would find an empty file where a credential has to be, which presents as a container +// that will not start, a long way from the cause. +func secretsByVariableIn(manifest []byte) (map[string]string, error) { var m struct { OwnSecrets map[string]string `json:"own-secrets"` Resources []struct { - Type string `json:"type"` - Env map[string]string `json:"env"` + Type string `json:"type"` + Path string `json:"path"` + Content string `json:"content"` + Env map[string]string `json:"env"` + Volumes []string `json:"volumes"` } `json:"resources"` } if err := json.Unmarshal(manifest, &m); err != nil { return nil, fmt.Errorf("the %s module's manifest is not readable: %w", ControlPlaneModule, err) } - byPath := map[string]string{} + secretAt := map[string]string{} for name, path := range m.OwnSecrets { - byPath[path] = name + secretAt[path] = name } wanted := map[string]string{} for _, r := range m.Resources { - if r.Type != "container" { - continue - } - for key, path := range r.Env { - if !strings.HasPrefix(key, storeVariablePrefix) || !strings.HasSuffix(key, storeFileSuffix) { - continue + switch r.Type { + case "file": + for variable, secret := range secretsInContent(r.Content) { + wanted[variable] = secret } - context := strings.TrimSuffix(strings.TrimPrefix(key, storeVariablePrefix), storeFileSuffix) - secret, ok := byPath[path] - if !ok { - return nil, fmt.Errorf( - "the %s module's container reads the %s store's connection from %s, and no "+ - "own-secret of that module writes that file.\n"+ - "So the mesh would seal nothing there and the control plane would find an "+ - "empty file where a connection string has to be. The manifest has to name "+ - "the two ends the same", - ControlPlaneModule, strings.ToLower(context), path) + case "container": + inside := mountedFrom(r.Volumes) + for key, path := range r.Env { + if !strings.HasPrefix(key, storeVariablePrefix) || + !strings.HasSuffix(key, storeFileSuffix) { + continue + } + on := path + if from, mounted := inside[path]; mounted { + on = from + } + secret, named := secretAt[on] + if !named { + return nil, fmt.Errorf( + "the %s module's container reads %s from %s, and no own-secret of that "+ + "module writes that file.\n"+ + "So the mesh would seal nothing there and the control plane would find "+ + "an empty file where a connection string has to be. The manifest has to "+ + "name the two ends the same, directly or through a mount", + ControlPlaneModule, key, path) + } + wanted[strings.TrimSuffix(key, storeFileSuffix)] = secret } - wanted[context] = secret } } return wanted, nil } +// mountedFrom is where each path inside a container comes from outside it. +func mountedFrom(volumes []string) map[string]string { + inside := map[string]string{} + for _, volume := range volumes { + parts := strings.Split(volume, ":") + if len(parts) < 2 { + continue + } + inside[parts[1]] = parts[0] + } + return inside +} + +// secretsInContent finds `VAR=${secret:name}` lines in a file the container reads its environment +// from. +func secretsInContent(content string) map[string]string { + found := map[string]string{} + for _, line := range strings.Split(content, "\n") { + variable, value, is := strings.Cut(strings.TrimSpace(line), "=") + if !is { + continue + } + const opens = "${secret:" + if !strings.HasPrefix(value, opens) || !strings.HasSuffix(value, "}") { + continue + } + found[variable] = strings.TrimSuffix(strings.TrimPrefix(value, opens), "}") + } + return found +} + +// anyStoreIn reports whether any of these variables is a context's connection. +func anyStoreIn(wanted map[string]string) bool { + for variable := range wanted { + if strings.HasPrefix(variable, storeVariablePrefix) { + return true + } + } + return false +} + func sortedKeys(m map[string]string) []string { keys := make([]string, 0, len(m)) for k := range m { diff --git a/internal/bootstrap/control_test.go b/internal/bootstrap/control_test.go index bf4d97c..9a6a5e1 100644 --- a/internal/bootstrap/control_test.go +++ b/internal/bootstrap/control_test.go @@ -11,28 +11,47 @@ import ( // that could go wrong quietly: pinning it to the wrong image, and delivering it store connections // the mesh invented rather than the ones the substrate actually made. -// theControlPlaneModule is the shape this installer codes against: one container using the -// catalogue's placeholder-digest convention, with each store connection delivered as a sealed -// secret written into a file and MESH_STORE__FILE pointing at it. +// theControlPlaneModule is the catalogue's manifest, trimmed to what this installer reads. +// +// A fixture rather than the file itself, unlike the substrate example the rewrite tests use: the +// catalogue is a different repository on a different branch, and a test that read it would pass or +// fail according to what somebody else had checked out. What it must stay faithful to is the +// SHAPE — the placeholder digest, the own-secret per context, the mount from the machine's path to +// the container's, and the environment file that fills what is not a path. const theControlPlaneModule = `{ "module": "mesh-control", "version": "1", + "slug": "control", "capabilities": ["container-runtime"], + "claims": [{"name": "the-control-plane", "scope": "mesh"}], "own-secrets": { - "inventory-store": "/var/lib/mesh/control/inventory", - "identity-store": "/var/lib/mesh/control/identity", - "licences-store": "/var/lib/mesh/control/licences" + "inventory": "/var/lib/mesh/mesh-control/inventory", + "identity": "/var/lib/mesh/mesh-control/identity", + "licences": "/var/lib/mesh/mesh-control/licences", + "broker": "/var/lib/mesh/mesh-control/broker", + "broker-management": "/var/lib/mesh/mesh-control/broker-management" }, "resources": [ - {"id": "state", "type": "directory", "path": "/var/lib/mesh/control", "mode": "0700"}, - {"id": "container", "type": "container", "name": "mesh-control", + {"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-control", "mode": "0700"}, + {"id": "broker-env", "type": "file", "path": "/var/lib/mesh/mesh-control/broker.env", + "mode": "0600", + "content": "MESH_BROKER_AMQP=${secret:broker}\nMESH_BROKER_MANAGEMENT=${secret:broker-management}\nMESH_BROKER_ADDRESS=${machine:at}:5671\n"}, + {"id": "server", "type": "container", "name": "mesh-control", "image": "mesh-control@` + placeholderDigest + `", "network": "host", "args": ["serve"], + "env-file": ["/var/lib/mesh/mesh-control/broker.env"], "env": { - "MESH_STORE_INVENTORY_FILE": "/var/lib/mesh/control/inventory", - "MESH_STORE_IDENTITY_FILE": "/var/lib/mesh/control/identity", - "MESH_STORE_LICENCES_FILE": "/var/lib/mesh/control/licences" - }} + "MESH_STORE_INVENTORY_FILE": "/run/secrets/inventory", + "MESH_STORE_IDENTITY_FILE": "/run/secrets/identity", + "MESH_STORE_LICENCES_FILE": "/run/secrets/licences", + "MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt" + }, + "volumes": [ + "mesh-broker-tls:/broker-tls:ro", + "/var/lib/mesh/mesh-control/inventory:/run/secrets/inventory:ro", + "/var/lib/mesh/mesh-control/identity:/run/secrets/identity:ro", + "/var/lib/mesh/mesh-control/licences:/run/secrets/licences:ro" + ]} ] }` @@ -76,8 +95,8 @@ func TestAManifestAlreadyPinnedByHandIsRefused(t *testing.T) { // otherwise be left half pinned, and fail inside an apply rather than here. func TestEveryPlaceTheManifestNamesTheImageIsPinned(t *testing.T) { twice := strings.Replace(theControlPlaneModule, - `{"id": "state", "type": "directory", "path": "/var/lib/mesh/control", "mode": "0700"},`, - `{"id": "state", "type": "directory", "path": "/var/lib/mesh/control", "mode": "0700"}, + `{"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-control", "mode": "0700"},`, + `{"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-control", "mode": "0700"}, {"id": "migrate", "type": "container", "name": "mesh-control-migrate", "run-once": true, "image": "mesh-control@`+placeholderDigest+`", "args": ["migrate"]},`, 1) @@ -99,22 +118,30 @@ func TestEveryPlaceTheManifestNamesTheImageIsPinned(t *testing.T) { // context. The pairing is read from the manifest so that whatever the catalogue calls these // secrets is what is delivered. func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) { - wanted, err := storeSecretsIn([]byte(theControlPlaneModule)) + wanted, err := secretsByVariableIn([]byte(theControlPlaneModule)) if err != nil { t.Fatal(err) } - for context, secret := range map[string]string{ - "INVENTORY": "inventory-store", - "IDENTITY": "identity-store", - "LICENCES": "licences-store", + // **Through the mount.** The manifest keeps its secrets under /var/lib and the container reads + // them at /run/secrets. Matching on the path alone would find nothing and refuse a correct + // manifest, which is exactly the ordinary case in the catalogue. + for variable, secret := range map[string]string{ + "MESH_STORE_INVENTORY": "inventory", + "MESH_STORE_IDENTITY": "identity", + "MESH_STORE_LICENCES": "licences", + "MESH_BROKER_AMQP": "broker", + "MESH_BROKER_MANAGEMENT": "broker-management", } { - if wanted[context] != secret { - t.Errorf("the %s store's connection would be accepted as %q, want %q", - context, wanted[context], secret) + if wanted[variable] != secret { + t.Errorf("%s would be accepted as %q, want %q", variable, wanted[variable], secret) } } + // And what the manifest fills from the machine rather than from a secret is left alone. + if _, claimed := wanted["MESH_BROKER_ADDRESS"]; claimed { + t.Error("the address the mesh composes from the machine was treated as a secret") + } - // And the values are the substrate's own, taken from the produced bundle rather than composed. + // The values are the substrate's own, taken from the produced bundle rather than composed. rewritten, err := Rewrite(theRealBundle(t), held) if err != nil { t.Fatal(err) @@ -127,8 +154,9 @@ func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) { if err != nil { t.Fatal(err) } - if len(delivered) != 3 { - t.Fatalf("%d connections were delivered, and the mesh holds three contexts: %v", + // Three stores and both halves of the broker: everything the substrate made and nothing else. + if len(delivered) != 5 { + t.Fatalf("%d values were delivered, and the substrate names five: %v", len(delivered), delivered) } for _, secret := range delivered { @@ -138,15 +166,49 @@ func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) { } } +// A secret the substrate did not make is left for the mesh to make, and said so. Every other +// secret in a mesh is one the mesh made; `secret accept` is only for what predates the mesh. +func TestASecretTheSubstrateNeverMadeIsLeftToTheMesh(t *testing.T) { + extra := strings.Replace(theControlPlaneModule, + `"broker": "/var/lib/mesh/mesh-control/broker",`, + `"broker": "/var/lib/mesh/mesh-control/broker", + "something-new": "/var/lib/mesh/mesh-control/something-new",`, 1) + extra = strings.Replace(extra, + `"content": "MESH_BROKER_AMQP=${secret:broker}\n`, + `"content": "MESH_SOMETHING_NEW=${secret:something-new}\nMESH_BROKER_AMQP=${secret:broker}\n`, 1) + + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + runtime := &asked{answer: aMeshThatAgrees(nil)} + control := controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second} + + var said []string + delivered, err := deliverStores(context.Background(), Options{Node: "anchor"}, control, + []byte(extra), rewritten.Declaration, func(line string) { said = append(said, line) }) + if err != nil { + t.Fatal(err) + } + for _, secret := range delivered { + if secret == "something-new" { + t.Error("a value the substrate never made was accepted as though it had") + } + } + if !strings.Contains(strings.Join(said, "\n"), "the mesh will make something-new") { + t.Errorf("nothing was said about the secret the mesh has to make: %v", said) + } +} + // A manifest whose container reads a file no own-secret writes is refused. The mesh would seal // nothing there and the control plane would find an empty file where a connection string has to // be — which presents as a control plane that will not start, three steps from the cause. func TestAConnectionFileNothingWritesIsRefused(t *testing.T) { mismatched := strings.Replace(theControlPlaneModule, - `"inventory-store": "/var/lib/mesh/control/inventory"`, - `"inventory-store": "/var/lib/mesh/control/somewhere-else"`, 1) + `"inventory": "/var/lib/mesh/mesh-control/inventory",`, + `"inventory": "/var/lib/mesh/mesh-control/somewhere-else",`, 1) - _, err := storeSecretsIn([]byte(mismatched)) + _, err := secretsByVariableIn([]byte(mismatched)) if err == nil { t.Fatal("a manifest whose two ends do not meet was accepted") }