package bootstrap import ( "bytes" "context" "encoding/json" "fmt" "sort" "strings" "github.com/novox/mesh-host/internal/declaration" ) // storeFileSuffix is how a manifest asks for a store connection in a file rather than in the // environment. // // `MESH_STORE_` is what the control plane reads (mesh-control's `internal/store`.Variable) // and putting a password in a container's environment puts it in `docker inspect` for ever. So a // module manifest names a file per context and points at it with `…_FILE`; the mesh seals the value // into that file on the machine, and nothing but the process reads it. const storeFileSuffix = "_FILE" // storeVariablePrefix is the front of the same names. const storeVariablePrefix = "MESH_STORE_" // Permanent is what step 9 did. type Permanent struct { Installed // Container is what the module calls its container, confirmed running. Container string // Image is what it is pinned to — the digest step 8's push produced. Image string // Delivered is every store connection accepted into it, by secret name. Delivered []string // Answered is what the permanent control plane said back. Answered string } // InstallControlPlane makes the control plane an ordinary module. // // **The host performs the replacement, not the control plane** (novox/hq ADR 0067). The temporary // control plane composes a declaration naming the registry-pinned image, publishes it, and this // node's host creates the container. Nothing is asked to replace itself while running, which is // what makes the whole thing expressible: the container being created is called `mesh-control` and // the one composing it is called `temp-mesh-control`, so there are two of them and neither is in // the other's way. // // **The store connections are the substrate's, made at genesis, and the mesh cannot invent them.** // Every other secret in a mesh is one the mesh made; these existed before the mesh did — they are // the credentials the substrate bundle created the databases with. Generating replacements would // put thirty-two random bytes where a working connection string has to be, and the control plane // would come up unable to open a single context. So they go in through `secret accept`, which is // exactly the path for a value the mesh must carry and could not have invented — and they are read // out of the bundle this installer produced rather than reconstructed, because the bundle is what // created them and a second opinion about what a DSN should say is a second chance to be wrong. func InstallControlPlane(ctx context.Context, o Options, d Deps, control controlPlane, substrate *declaration.Declaration, image string, say func(string)) (Permanent, error) { out := Permanent{Image: image} manifest, err := readManifest(o.Catalogue, ControlPlaneModule) if err != nil { return out, fmt.Errorf( "%w\n"+ "This is the manifest that makes the control plane an ordinary module. Without it "+ "the machine keeps the temporary control plane the substrate raised, which works "+ "and cannot be upgraded — so the install stops here rather than pretending to "+ "have pivoted", err) } pinned, places, err := pinImage(manifest, image) if err != nil { return out, err } say(fmt.Sprintf(" pinned %s, in %d place(s)", image, places)) container, _, err := containerIn(pinned, controlPlaneResourceIn(pinned)) if err != nil { return out, err } out.Container = container if container == "" { return out, fmt.Errorf( "the %s module's container has no name, so nothing can be verified afterwards", ControlPlaneModule) } installed, err := registerAndAssign(ctx, o, control, ControlPlaneModule, pinned, say) out.Installed = installed if err != nil { return out, err } // The connections, before the push that would otherwise deliver random bytes for them. delivered, err := deliverStores(ctx, o, control, pinned, substrate, say) out.Delivered = delivered if err != nil { return out, err } if out.Pushed, err = pushNode(ctx, o, control, say); err != nil { return out, err } if err := waitForContainer(ctx, control.run, o.Timeout, o.Wait, container, say); err != nil { return out, err } // And it answers, which is the same question step 5 asked of the temporary one and for the // same reason: `status` opens all three stores, so a reply proves the sealed connections it // was given are the ones the substrate made. Asked of the NEW container — this is the only // moment in the program where two control planes are running, and asking the wrong one would // report the temporary one's health as the permanent one's. answered, err := waitForTheControlPlane(ctx, control.run, o.Timeout, o.Wait, container, say) if err != nil { return out, err } out.Answered = answered return out, nil } // pinImage replaces the catalogue's placeholder digest with what the registry assigned. // // **Textual, and every place it appears.** A manifest may name its image in more than one resource // — the catalogue's converted modules routinely carry a runtime container beside the application's // — and the same reasoning as the bundle rewrite applies: replacing one and not the others leaves // something pointing at an image nothing serves, and it fails half way through an apply rather // than here. // // It refuses a manifest with no placeholder in it. That is not pedantry: a manifest already // carrying a real digest is one somebody pinned by hand, and quietly registering it would install a // control plane that is not the image this machine just published — which is the one thing this // step exists to guarantee. func pinImage(manifest []byte, reference string) ([]byte, int, error) { places := bytes.Count(manifest, []byte(placeholderDigest)) if places == 0 { return nil, 0, fmt.Errorf( "the %s module's manifest carries no placeholder digest (%s), so there is nothing to "+ "pin to the image this machine just published.\n"+ "A manifest already naming a digest was pinned by somebody else, to some other "+ "build. Registering it would install a control plane that is not the one this "+ "installer carried and pushed", ControlPlaneModule, placeholderDigest) } // The reference the registry gave back is `/@sha256:…`, and what the // manifest holds is `@sha256:0…0`. Replacing only the digest would leave the // manifest's own repository name in front of it — which may be `mesh-control` with no // registry, and a runtime would then pull it from the internet. The whole reference moves. var out bytes.Buffer rest := manifest for { at := bytes.Index(rest, []byte(placeholderDigest)) if at < 0 { out.Write(rest) break } // Back up over the repository this digest belongs to, which runs to the opening quote. start := bytes.LastIndexByte(rest[:at], '"') if start < 0 { return nil, 0, fmt.Errorf( "the %s module's manifest has a placeholder digest that is not inside a JSON "+ "string, so the installer cannot tell what image it belongs to", ControlPlaneModule) } out.Write(rest[:start+1]) out.WriteString(reference) rest = rest[at+len(placeholderDigest):] } pinned := out.Bytes() // Read back. A substitution on text can catch more than it was aimed at, and the manifest is // about to be handed to the mesh as the description of what it runs. var checked map[string]any if err := json.Unmarshal(pinned, &checked); err != nil { return nil, 0, fmt.Errorf( "pinning the %s module's image broke its manifest: %w", ControlPlaneModule, err) } if bytes.Contains(pinned, []byte(placeholderDigest)) { return nil, 0, fmt.Errorf( "the %s module's manifest still carries a placeholder digest after pinning", ControlPlaneModule) } return pinned, places, nil } // controlPlaneResourceIn is the id of the resource that runs the control plane. // // The manifest is written by the catalogue and the installer does not get to name its resources. // What it can do is find the one container whose image is the one just pinned — and when a manifest // declares exactly one container, that is the answer without any searching at all. func controlPlaneResourceIn(manifest []byte) string { var m struct { Resources []struct { ID string `json:"id"` Type string `json:"type"` } `json:"resources"` } if err := json.Unmarshal(manifest, &m); err != nil { return "" } var containers []string for _, r := range m.Resources { if r.Type == "container" { containers = append(containers, r.ID) } } if len(containers) == 1 { return containers[0] } // More than one, so the name has to be guessed at rather than derived — and the catalogue's // own convention for the resource that IS the module is `container`, with anything else beside // it named for what it does. for _, id := range containers { if id == "container" || id == ControlPlaneModule || id == "control-plane" { return id } } return "" } // deliverStores carries the substrate's own database connections into the module. // // The pairing is read from the manifest rather than assumed, so that whatever the catalogue calls // these secrets is what is delivered: a container asking for `MESH_STORE_INVENTORY_FILE` names a // path, and the module's own-secret that writes that path is the secret to accept the connection // as. That is one lookup and it cannot get the wrong secret — the alternative, guessing that the // secret is called `inventory`, would seal a connection string under a name nothing reads and // leave the mesh to invent random bytes for the one that is. func deliverStores(ctx context.Context, o Options, control controlPlane, manifest []byte, substrate *declaration.Declaration, say func(string)) ([]string, error) { wanted, err := storeSecretsIn(manifest) if err != nil { return nil, err } if len(wanted) == 0 { return nil, fmt.Errorf( "the %s module's manifest asks for no store connections. A control plane reaches each "+ "context through its own credential (novox/hq ADR 0008), so a manifest naming none "+ "describes a control plane that can open nothing.\n"+ "The shape this installer delivers into is a file per context, named by an "+ "own-secret, with %s%s in the container's environment pointing at it", ControlPlaneModule, storeVariablePrefix, storeFileSuffix) } temporary, err := controlPlaneIn(substrate) if err != nil { return nil, err } var delivered []string for _, context := range sortedKeys(wanted) { secret := wanted[context] connection := temporary.Env[storeVariablePrefix+context] if strings.TrimSpace(connection) == "" { return delivered, fmt.Errorf( "the %s module wants the %s store's connection and the bundle this installer "+ "produced does not name one: its control plane has no %s.\n"+ "These connections are the substrate's, created at genesis — the mesh cannot "+ "invent them and the installer will not guess at one", ControlPlaneModule, strings.ToLower(context), storeVariablePrefix+context) } // Into the container as a file, because `secret accept` reads a file or a prompt and the // installer has neither a terminal to be prompted at nor a way to write to a command's // standard input through the runner every applier in this repository shares. at := "/accepting-" + strings.ToLower(context) if err := control.carrying(ctx, "mesh-store-"+strings.ToLower(context), []byte(connection), at); err != nil { return delivered, err } if _, err := control.tell(ctx, "secret", "accept", o.Node, ControlPlaneModule, secret, "--from", at); err != nil { return delivered, err } delivered = append(delivered, secret) say(" accepted " + secret + " — the " + strings.ToLower(context) + " store, as the substrate made it") } return delivered, nil } // storeSecretsIn pairs each context with the secret its connection must be accepted as. // // Read out of the manifest twice over: the container's environment says which contexts are wanted // and what file each expects, and the module's own-secrets say which secret writes which file. A // pair that does not meet is refused rather than half-delivered. func storeSecretsIn(manifest []byte) (map[string]string, error) { var m struct { OwnSecrets map[string]string `json:"own-secrets"` Resources []struct { Type string `json:"type"` Env map[string]string `json:"env"` } `json:"resources"` } if err := json.Unmarshal(manifest, &m); err != nil { return nil, fmt.Errorf("the %s module's manifest is not readable: %w", ControlPlaneModule, err) } byPath := map[string]string{} for name, path := range m.OwnSecrets { byPath[path] = name } wanted := map[string]string{} for _, r := range m.Resources { if r.Type != "container" { continue } for key, path := range r.Env { if !strings.HasPrefix(key, storeVariablePrefix) || !strings.HasSuffix(key, storeFileSuffix) { continue } context := strings.TrimSuffix(strings.TrimPrefix(key, storeVariablePrefix), storeFileSuffix) secret, ok := byPath[path] if !ok { return nil, fmt.Errorf( "the %s module's container reads the %s store's connection from %s, and no "+ "own-secret of that module writes that file.\n"+ "So the mesh would seal nothing there and the control plane would find an "+ "empty file where a connection string has to be. The manifest has to name "+ "the two ends the same", ControlPlaneModule, strings.ToLower(context), path) } wanted[context] = secret } } return wanted, nil } func sortedKeys(m map[string]string) []string { keys := make([]string, 0, len(m)) for k := range m { keys = append(keys, k) } sort.Strings(keys) return keys }