package bootstrap import ( "bytes" "context" "encoding/json" "fmt" "sort" "strings" "github.com/novox/mesh-host/internal/declaration" ) // storeFileSuffix is how a manifest asks for a store connection in a file rather than in the // environment. // // `MESH_STORE_` is what the control plane reads (mesh-controller's `internal/store`.Variable) // and putting a password in a container's environment puts it in `docker inspect` for ever. So a // module manifest names a file per context and points at it with `…_FILE`; the mesh seals the value // into that file on the machine, and nothing but the process reads it. const storeFileSuffix = "_FILE" // storeVariablePrefix is the front of the same names. const storeVariablePrefix = "MESH_STORE_" // Permanent is what step 9 did. type Permanent struct { Installed // Container is what the module calls its container, confirmed running. Container string // Image is what it is pinned to — the digest step 8's push produced. Image string // Delivered is every store connection accepted into it, by secret name. Delivered []string // Answered is what the permanent control plane said back. Answered string } // InstallControlPlane makes the control plane an ordinary module. // // **The host performs the replacement, not the control plane** (novox/hq ADR 0067). The temporary // control plane composes a declaration naming the registry-pinned image, publishes it, and this // node's host creates the container. Nothing is asked to replace itself while running, which is // what makes the whole thing expressible: the container being created is called `mesh-controller` and // the one composing it is called `temp-mesh-controller`, so there are two of them and neither is in // the other's way. // // **The store connections are the foundation's, made at genesis, and the mesh cannot invent them.** // Every other secret in a mesh is one the mesh made; these existed before the mesh did — they are // the credentials the foundation bundle created the databases with. Generating replacements would // put thirty-two random bytes where a working connection string has to be, and the control plane // would come up unable to open a single context. So they go in through `secret accept`, which is // exactly the path for a value the mesh must carry and could not have invented — and they are read // out of the bundle this installer produced rather than reconstructed, because the bundle is what // created them and a second opinion about what a DSN should say is a second chance to be wrong. func InstallControlPlane(ctx context.Context, o Options, d Deps, control controlPlane, foundation *declaration.Declaration, image string, say func(string)) (Permanent, error) { out := Permanent{Image: image} manifest, err := readManifest(o.Catalogue, ControlPlaneModule) if err != nil { return out, fmt.Errorf( "%w\n"+ "This is the manifest that makes the control plane an ordinary module. Without it "+ "the machine keeps the temporary control plane the foundation raised, which works "+ "and cannot be upgraded — so the install stops here rather than pretending to "+ "have pivoted", err) } pinned, places, err := pinImage(manifest, image, ControlPlaneModule) if err != nil { return out, err } say(fmt.Sprintf(" pinned %s, in %d place(s)", image, places)) container, _, err := containerIn(pinned, controlPlaneResourceIn(pinned)) if err != nil { return out, err } out.Container = container if container == "" { return out, fmt.Errorf( "the %s module's container has no name, so nothing can be verified afterwards", ControlPlaneModule) } installed, err := registerAndAssign(ctx, o, control, ControlPlaneModule, pinned, say) out.Installed = installed if err != nil { return out, err } // The connections, before the push that would otherwise deliver random bytes for them. delivered, err := deliverStores(ctx, o, control, pinned, foundation, say) out.Delivered = delivered if err != nil { return out, err } if out.Pushed, err = pushNode(ctx, o, control, say); err != nil { return out, err } if err := waitForContainer(ctx, control.run, o.Timeout, o.Wait, container, say); err != nil { return out, err } // And it answers, which is the same question step 5 asked of the temporary one and for the // same reason: `status` opens all three stores, so a reply proves the sealed connections it // was given are the ones the foundation made. Asked of the NEW container — this is the only // moment in the program where two control planes are running, and asking the wrong one would // report the temporary one's health as the permanent one's. answered, err := waitForTheControlPlane(ctx, control.run, o.Timeout, o.Wait, container, say) if err != nil { return out, err } out.Answered = answered return out, nil } // pinImage replaces the catalogue's placeholder digest with what the registry assigned. // // **Textual, and every place it appears.** A manifest may name its image in more than one resource // — the catalogue's converted modules routinely carry a runtime container beside the application's // — and the same reasoning as the bundle rewrite applies: replacing one and not the others leaves // something pointing at an image nothing serves, and it fails half way through an apply rather // than here. // // It refuses a manifest with no placeholder in it. That is not pedantry: a manifest already // carrying a real digest is one somebody pinned by hand, and quietly registering it would install a // control plane that is not the image this machine just published — which is the one thing this // step exists to guarantee. func pinImage(manifest []byte, reference, module string) ([]byte, int, error) { places := bytes.Count(manifest, []byte(placeholderDigest)) if places == 0 { return nil, 0, fmt.Errorf( "the %s module's manifest carries no placeholder digest (%s), so there is nothing to "+ "pin to the image this machine just published.\n"+ "A manifest already naming a digest was pinned by somebody else, to some other "+ "build. Registering it would install a module that is not the one this "+ "installer carried and pushed", module, placeholderDigest) } // The reference the registry gave back is `/@sha256:…`, and what the // manifest holds is `@sha256:0…0`. Replacing only the digest would leave the // manifest's own repository name in front of it — which may be `mesh-controller` with no // registry, and a runtime would then pull it from the internet. The whole reference moves. var out bytes.Buffer rest := manifest for { at := bytes.Index(rest, []byte(placeholderDigest)) if at < 0 { out.Write(rest) break } // Back up over the repository this digest belongs to, which runs to the opening quote. start := bytes.LastIndexByte(rest[:at], '"') if start < 0 { return nil, 0, fmt.Errorf( "the %s module's manifest has a placeholder digest that is not inside a JSON "+ "string, so the installer cannot tell what image it belongs to", module) } out.Write(rest[:start+1]) out.WriteString(reference) rest = rest[at+len(placeholderDigest):] } pinned := out.Bytes() // Read back. A substitution on text can catch more than it was aimed at, and the manifest is // about to be handed to the mesh as the description of what it runs. var checked map[string]any if err := json.Unmarshal(pinned, &checked); err != nil { return nil, 0, fmt.Errorf( "pinning the %s module's image broke its manifest: %w", module, err) } if bytes.Contains(pinned, []byte(placeholderDigest)) { return nil, 0, fmt.Errorf( "the %s module's manifest still carries a placeholder digest after pinning", module) } return pinned, places, nil } // controlPlaneResourceIn is the id of the resource that runs the control plane. // // The manifest is written by the catalogue and the installer does not get to name its resources. // What it can do is find the one container whose image is the one just pinned — and when a manifest // declares exactly one container, that is the answer without any searching at all. func controlPlaneResourceIn(manifest []byte) string { var m struct { Resources []struct { ID string `json:"id"` Type string `json:"type"` } `json:"resources"` } if err := json.Unmarshal(manifest, &m); err != nil { return "" } var containers []string for _, r := range m.Resources { if r.Type == "container" { containers = append(containers, r.ID) } } if len(containers) == 1 { return containers[0] } // More than one, so the name has to be guessed at rather than derived — and the catalogue's // own convention for the resource that IS the module is `container`, with anything else beside // it named for what it does. for _, id := range containers { if id == "container" || id == ControlPlaneModule || id == "control-plane" { return id } } return "" } // deliverStores carries the foundation's own database connections into the module. // // The pairing is read from the manifest rather than assumed, so that whatever the catalogue calls // these secrets is what is delivered. The installer does not guess that the secret holding the // inventory connection is called `inventory`; it follows the manifest from the variable to the // secret, and a manifest whose two ends do not meet is refused rather than half-delivered. // // **What is delivered is what the foundation already has, and only that.** The mesh generates an // own-secret nobody supplied, which is right for something coming into existence and wrong for // something that already exists. So every variable the module fills from a secret is looked up in // the foundation's control plane: what it names is accepted, what it does not is left for the mesh // to make. A store connection missing from the foundation is the one exception and is an error — // a control plane that cannot open a context is not a control plane. func deliverStores(ctx context.Context, o Options, control controlPlane, manifest []byte, foundation *declaration.Declaration, say func(string)) ([]string, error) { wanted, err := secretsByVariableIn(manifest) if err != nil { return nil, err } if !anyStoreIn(wanted) { return nil, fmt.Errorf( "the %s module's manifest fills no %s… variable from a secret. A control plane reaches "+ "each context through its own credential (novox/hq ADR 0008), so a manifest naming "+ "none describes a control plane that can open nothing.\n"+ "The shape this installer delivers into is a file per context, named by an "+ "own-secret, with %s%s pointing at it — directly, or at where that file is "+ "mounted inside the container", ControlPlaneModule, storeVariablePrefix, storeVariablePrefix, storeFileSuffix) } temporary, err := controlPlaneIn(foundation) if err != nil { return nil, err } var delivered []string for _, variable := range sortedKeys(wanted) { secret := wanted[variable] value := strings.TrimSpace(temporary.Env[variable]) if value == "" { if strings.HasPrefix(variable, storeVariablePrefix) { return delivered, fmt.Errorf( "the %s module wants %s and the bundle this installer produced does not name "+ "one.\n"+ "That connection is the foundation's, created at genesis — the mesh cannot "+ "invent it and the installer will not guess at one", ControlPlaneModule, variable) } // Not something the foundation made. The mesh generates its own, which is exactly what // an own-secret is for; said so that nothing about the delivery is silent. say(" the mesh will make " + secret + " — the foundation names no " + variable) continue } // Into the container as a file, because `secret accept` reads a file or a prompt and the // installer has neither a terminal to be prompted at nor a way to write to a command's // standard input through the runner every applier in this repository shares. at := "/accepting-" + secret if err := control.carryingSecret(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil { return delivered, err } if _, err := control.tell(ctx, "secret", "accept", o.Node, ControlPlaneModule, secret, "--from", at); err != nil { return delivered, err } delivered = append(delivered, secret) say(" accepted " + secret + " — " + variable + ", as the foundation made it") } return delivered, nil } // secretsByVariableIn maps each environment variable the module fills from a secret to that // secret's name. // // Two shapes, because the catalogue uses both: // // - `MESH_STORE__FILE` in the container's environment, naming a path the process reads. // The path may be the own-secret's own path, or — more usually — where that file is mounted // inside the container, in which case the volumes say which is which. Following the mount is // not a nicety: a manifest that keeps its secrets under `/var/lib/mesh/…` and mounts them at // `/run/secrets/…` is the ordinary case, and matching on the path alone would find nothing and // refuse a correct manifest. // - `VAR=${secret:name}` inside a file resource the container reads its environment from, which // is how a value that is not a path gets in at all. // // A `…_FILE` variable whose file nothing writes is refused: the mesh would seal nothing there and // the process would find an empty file where a credential has to be, which presents as a container // that will not start, a long way from the cause. func secretsByVariableIn(manifest []byte) (map[string]string, error) { var m struct { OwnSecrets map[string]string `json:"own-secrets"` Resources []struct { Type string `json:"type"` Path string `json:"path"` Content string `json:"content"` Env map[string]string `json:"env"` Volumes []string `json:"volumes"` } `json:"resources"` } if err := json.Unmarshal(manifest, &m); err != nil { return nil, fmt.Errorf("the %s module's manifest is not readable: %w", ControlPlaneModule, err) } secretAt := map[string]string{} for name, path := range m.OwnSecrets { secretAt[path] = name } wanted := map[string]string{} for _, r := range m.Resources { switch r.Type { case "file": for variable, secret := range secretsInContent(r.Content) { wanted[variable] = secret } case "container": inside := mountedFrom(r.Volumes) for key, path := range r.Env { // Any `MESH_…_FILE` naming an own-secret's file, not only the store's: the broker // settings took the same shape once a secret stopped travelling in an env-file // (novox/hq ADR 0086, issue 041). if !strings.HasPrefix(key, "MESH_") || !strings.HasSuffix(key, storeFileSuffix) { continue } on := path if from, mounted := inside[path]; mounted { on = from } secret, named := secretAt[on] if !named { return nil, fmt.Errorf( "the %s module's container reads %s from %s, and no own-secret of that "+ "module writes that file.\n"+ "So the mesh would seal nothing there and the control plane would find "+ "an empty file where a connection string has to be. The manifest has to "+ "name the two ends the same, directly or through a mount", ControlPlaneModule, key, path) } wanted[strings.TrimSuffix(key, storeFileSuffix)] = secret } } } return wanted, nil } // mountedFrom is where each path inside a container comes from outside it. func mountedFrom(volumes []string) map[string]string { inside := map[string]string{} for _, volume := range volumes { parts := strings.Split(volume, ":") if len(parts) < 2 { continue } inside[parts[1]] = parts[0] } return inside } // secretsInContent finds `VAR=${secret:name}` lines in a file the container reads its environment // from. func secretsInContent(content string) map[string]string { found := map[string]string{} for _, line := range strings.Split(content, "\n") { variable, value, is := strings.Cut(strings.TrimSpace(line), "=") if !is { continue } const opens = "${secret:" if !strings.HasPrefix(value, opens) || !strings.HasSuffix(value, "}") { continue } found[variable] = strings.TrimSuffix(strings.TrimPrefix(value, opens), "}") } return found } // anyStoreIn reports whether any of these variables is a context's connection. func anyStoreIn(wanted map[string]string) bool { for variable := range wanted { if strings.HasPrefix(variable, storeVariablePrefix) { return true } } return false } func sortedKeys(m map[string]string) []string { keys := make([]string, 0, len(m)) for k := range m { keys = append(keys, k) } sort.Strings(keys) return keys }