package bootstrap import ( "context" "encoding/json" "fmt" "net/http" "strings" "time" "github.com/novox/mesh-host/internal/declaration" ) // Phase three — nothing is special after installation (novox/hq issue 051). // // Genesis raises a store and a broker before any module system exists, because the control plane // cannot ask provisioning for the store it keeps its own records in or the broker it is reached over // (novox/hq ADR 0006). That leaves two servers behind: the foundation's, and a second one the // `postgres` and bus modules used to raise for other modules to use. This turns the foundation's // own servers into those modules, so a mesh runs ONE postgres and ONE bus — the control plane's // contexts and the database of each module that asks for one, in the same server (WBS 3.1/3.2). // // **Adopted in place, not replaced.** The control plane is stateless and is swapped for a fresh // container (control.go); the store and broker hold the mesh's memory and its bus, so they are kept. // The module declares a container with the same name, image and spec the foundation raised, and the // applier — which keys on the container name and compares a spec digest (mesh-host internal/apply) — // finds it already running and leaves it be. The image is pinned to the one the foundation is // running, read from the bundle this installer produced, so the two specs are the same digest and // nothing is recreated. A recreate happens only on a real upgrade, which is where a stated window // belongs (WBS 3.3). // StoreID and BrokerID are what the foundation bundle calls the two servers it raises; the modules // that adopt them are found by these ids in the bundle this installer produced, the same way the // control plane's own container is (ControlPlaneID). const ( StoreID = "store" BrokerID = "broker" ) // InstallStore makes the foundation's store the `postgres` module, adopted in place. // // The order is InstallFromCatalogue's, with two additions the store needs and an ordinary provider // does not: the server image is pinned to the one the foundation is already running (so the module's // container is the same spec and is adopted, not a second one raised), and the superuser password — // the foundation's, made at genesis — is carried in through `secret accept`, because the mesh cannot // invent a credential that already created the databases (the same reasoning as the control plane's // store connections, control.go deliverStores). func InstallStore(ctx context.Context, o Options, control controlPlane, foundation *declaration.Declaration, say func(string)) error { const module = "postgres" manifest, err := readManifest(o.Catalogue, module) if err != nil { return err } store, err := storeIn(foundation) if err != nil { return err } // The module adopts the running store rather than raising a second one, so its server container // has to BE the foundation's — same name, same image. The applier keys on the name and compares // a spec digest (internal/apply), so a mismatch here would not adopt the mesh's memory but // replace it. Checked before anything is registered, so a drift between the two pinned upstream // images (the catalogue's and the foundation bundle's) fails fast and by name, rather than // surfacing as the mesh's store being torn down and recreated. // // Not rewritten to the foundation's: the server image travels through the builder, which reads // the manifest from the repository and leaves a concrete image alone but would carry a rewrite // nowhere. The two are kept equal at the source — one pinned postgres, named in both places. if err := serverMatchesFoundation(manifest, store, module); err != nil { return err } say(" adopting " + store.Name + " — the store the foundation raised, unchanged") // The superuser is the foundation's, made at genesis — carried in before the push, or the push // would seal random bytes where a working password has to be and the provisioner would not open // the store it is meant to manage. if err := installProvider(ctx, o, control, module, manifest, "(the provisioner; the server is adopted, not built)", func() error { return deliverCredential(ctx, o, control, module, "superuser", StoreSuperuserFile, "the store's superuser", say) }, say); err != nil { return err } say(" adopted " + store.Name + " — the foundation's store is now the " + module + " module") return nil } // installProvider registers, builds, issues, assigns and pushes one catalogue module, with one // thing done just before the push — the moment a credential the mesh could not have made has to // be in it. InstallFromCatalogue is the same sequence without that moment; the store, the broker // and the vault each need it or need the shape, and three copies of it drifted. func installProvider(ctx context.Context, o Options, control controlPlane, module string, manifest []byte, buildNote string, beforePush func() error, say func(string), send ...func() error) error { remote := "/" + module + "-module.json" if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil { return err } if _, err := control.tell(ctx, "module", "add", remote); err != nil { return err } say(" registered " + module) if o.CatalogSource.Repository == "" { return fmt.Errorf("%s has to be built and there is no --catalog-source to build it from: "+ "the catalogue CHECKOUT says what it is, the catalogue REPOSITORY is where a builder "+ "clones it", module) } say(strings.TrimRight(" building "+module+" "+buildNote, " ")) if _, err := control.within(buildWait).tell(ctx, "build", o.CatalogSource.Repository, "--path", "modules/"+module, "--ref", refOr(o.CatalogSource.Ref), "--wait", "1200s"); err != nil { return err } if _, err := control.tell(ctx, "module", "issue", module, "--node", o.Node); err != nil { say(" no account " + module + " — it declares nothing to say on the broker") } else { say(" account issued " + module) } if _, err := control.tell(ctx, "assign", o.Node, module); err != nil { return err } if err := prepareModule(ctx, o, control, module, say); err != nil { return err } if beforePush != nil { if err := beforePush(); err != nil { return err } } // A module whose first send is not a plain push says how it is sent: the bus, whose replacement is a // planned step (ADR 0236), is sent by that step (adoptTheBus). if len(send) > 0 && send[0] != nil { return send[0]() } _, err := pushNode(ctx, o, control, say) return err } // deliverCredential carries a credential genesis made into a module as its own secret, through // `secret accept`: the mesh cannot invent the value a running server already has. func deliverCredential(ctx context.Context, o Options, control controlPlane, module, secret, file, what string, say func(string)) error { value, err := readCredentialFile(file) if err != nil { return fmt.Errorf("%s is not at %s, so the %s module has nothing to open its server with — "+ "and the mesh cannot invent the one the server already has: %w", what, file, module, err) } at := "/accepting-" + secret if err := control.carryingSecret(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil { return err } if _, err := control.tell(ctx, "secret", "accept", o.Node, module, secret, "--from", at); err != nil { return err } say(" accepted " + secret + " — " + what + ", as genesis made it") return nil } // BusModule is the module the foundation's bus is adopted as (novox/hq ADR 0331 §1). The previous // broker's module, lavinmq, left the catalogue with the move to one bus (ADR 0131). const BusModule = "nats" // InstallBroker makes the foundation's bus the `nats` module, adopted in place — the same shape as // InstallStore, for the same reasons: the bus holds the mesh's streams, so it is kept, and from the // module's first push the bus's user list reaches it in the module's declaration like any module's // (ADR 0331 §1, §5). The installer's own placing of that list (enrol.go placeTheBusUsers) was the // bridge to here and is not run again. // // **The bundle raises the bus where the module keeps it** (novox/hq ADR 0333 §1): the same container // name, the same streams directory, configuration directory, user list and certificate. That is checked // here against the module's own manifest before anything is registered, and a difference is refused // naming each one — under another name or on other paths the module would raise a second bus beside the // first, or an empty one in its place. // // **The image is the one thing that changes, once** (ADR 0333 §3). The store's module pins the upstream // postgres the foundation runs; the bus's module builds its server image (it carries the mesh's snapshot // tool), so no pinned reference can be equal to the foundation's. The applier recreates the container on // the module's image, on the same streams, users and certificate. The mesh replaces its bus only as a // planned step (ADR 0236), so the module is sent by that step: its streams snapshotted first, the restart // said, and the installer waits until the container runs the module's image, the bus answers, and the // controller ends the step done (bus_adoption.go), so nothing after this step dials a bus that is restarting. func InstallBroker(ctx context.Context, o Options, control controlPlane, foundation *declaration.Declaration, fetch func(context.Context, string) (int, string, error), say func(string)) error { const module = BusModule manifest, err := readManifest(o.Catalogue, module) if err != nil { return err } bus, err := brokerIn(foundation) if err != nil { return err } if err := busLayoutIsTheModules(manifest, foundation, module); err != nil { return err } say(" adopting " + bus.Name + " — the bus the foundation raised, its streams kept") if err := installProvider(ctx, o, control, module, manifest, "(the server, adopting the foundation's bus)", nil, say, func() error { return adoptTheBus(ctx, o, control, bus, say) }); err != nil { return err } if err := waitForTheAdoptedBus(ctx, o, control, bus, fetch, say); err != nil { return err } if err := waitForTheBusStep(ctx, o, control, say); err != nil { return err } say(" adopted " + bus.Name + " — the foundation's bus is now the " + module + " module, and its user list is the module's from here on") return nil } // busMonitor is where the bus says it is healthy: its monitoring port, published on the machine's own // loopback by the bundle and by the module alike (both "127.0.0.1:8222:8222"). /healthz answers ok only // once the server is up and JetStream has its streams back. const busMonitor = "http://127.0.0.1:8222/healthz" // runningImage is the image the named container was created from, or "" when there is none. func runningImage(ctx context.Context, control controlPlane, name string) string { asking, cancel := context.WithTimeout(ctx, control.timeout) defer cancel() out, err := control.run(asking, "docker", "inspect", "--format", "{{.Config.Image}}", name) if err != nil { return "" } return strings.TrimSpace(out) } // waitForTheAdoptedBus waits until the bus container runs an image other than the one the bundle raised // it from — the module's, after the applier's one recreate — and the bus answers on its monitor. // // Bounded by --wait, and asked at least once. A bus still on the bundle's image when it runs out is // refused, naming what it runs: the push was sent and the node has not applied it, and every step after // this one reaches the mesh through that bus. func waitForTheAdoptedBus(ctx context.Context, o Options, control controlPlane, bus *declaration.Container, fetch func(context.Context, string) (int, string, error), say func(string)) error { deadline := time.Now().Add(o.Wait) var running, said string for { running = runningImage(ctx, control, bus.Name) if running != "" && running != bus.Image { code, body, err := fetch(ctx, busMonitor) if err == nil && code == http.StatusOK { say(" bus answers " + bus.Name + " on " + running) return nil } said = strings.TrimSpace(body) if err != nil { said = err.Error() } } if !time.Now().Before(deadline) { break } select { case <-ctx.Done(): return ctx.Err() case <-time.After(answerEvery): } } switch { case running == "": return fmt.Errorf("the bus container %s is not there after the %s module was pushed, so nothing "+ "after this step could reach the mesh. The node-engine's journal says why it did not raise it", bus.Name, BusModule) case running == bus.Image: return fmt.Errorf("%s still runs the bundle's image %s %s after the %s module was pushed: the node "+ "has not applied it. Its streams are untouched; run this installer again once the node-engine "+ "applies (its journal says why it did not)", bus.Name, bus.Image, o.Wait, BusModule) } return fmt.Errorf("%s runs the %s module's image %s and its monitor (%s) did not answer ok within %s: %s", bus.Name, BusModule, running, busMonitor, o.Wait, orUnknown(said)) } // BusUsersID is the bundle's resource for the bus's first user list. const BusUsersID = "bus-accounts" // busLayout is where a bus lives: its container, the machine's directories it mounts as its streams, // its configuration and its certificate, and the file its user list is. type busLayout struct { Container, Streams, Conf, Certificate, Users string } // Where the server reads each, inside its container: the same in the bundle and the module, since both // run the same server with the same configuration. const ( busStreamsInside = "/data" busConfInside = "/etc/nats" busCertificateInside = "/tls" ) // foundationBusLayout reads the layout of the bus the bundle raises. func foundationBusLayout(foundation *declaration.Declaration) (busLayout, error) { bus, err := brokerIn(foundation) if err != nil { return busLayout{}, err } at := mountedFrom(bus.Volumes) l := busLayout{Container: bus.Name, Streams: at[busStreamsInside], Conf: at[busConfInside], Certificate: at[busCertificateInside]} for _, r := range foundation.Resources { if f, ok := r.(*declaration.File); ok && r.Identity() == BusUsersID { l.Users = f.Path } } return l, nil } // moduleBusLayout reads the layout of the bus the module declares: its server container, the mounts of // that container with every `${access:}` resolved to the access's path, and its `bus-users`. func moduleBusLayout(manifest []byte, module string) (busLayout, error) { var m struct { BusUsers string `json:"bus-users"` Resources []struct { ID string `json:"id"` Type string `json:"type"` Name string `json:"name"` Volumes []string `json:"volumes"` } `json:"resources"` Accesses []struct { ID string `json:"id"` Path string `json:"path"` } `json:"accesses"` } if err := json.Unmarshal(manifest, &m); err != nil { return busLayout{}, fmt.Errorf("the %s module's manifest is not readable: %w", module, err) } var containers []int for i, r := range m.Resources { if r.Type == "container" { containers = append(containers, i) } } if len(containers) != 1 { var names []string for _, i := range containers { names = append(names, m.Resources[i].Name) } return busLayout{}, fmt.Errorf("the %s module declares %s; the bus it adopts is one server, so it "+ "declares one container", module, describeNames(names)) } server := m.Resources[containers[0]] resolved := make([]string, 0, len(server.Volumes)) for _, v := range server.Volumes { for _, a := range m.Accesses { v = strings.ReplaceAll(v, "${access:"+a.ID+"}", a.Path) } resolved = append(resolved, v) } at := mountedFrom(resolved) return busLayout{Container: server.Name, Streams: at[busStreamsInside], Conf: at[busConfInside], Certificate: at[busCertificateInside], Users: m.BusUsers}, nil } // busLayoutIsTheModules checks that the bus the bundle raised is where the module keeps it, field by // field, so the applier takes it over rather than raising a second bus beside it or an empty one in its // place (novox/hq ADR 0333 §1). Every difference is named, with both sides. func busLayoutIsTheModules(manifest []byte, foundation *declaration.Declaration, module string) error { theirs, err := moduleBusLayout(manifest, module) if err != nil { return err } ours, err := foundationBusLayout(foundation) if err != nil { return err } var differ []string for _, f := range []struct{ what, module, bundle string }{ {"the server container", theirs.Container, ours.Container}, {"its streams (" + busStreamsInside + ")", theirs.Streams, ours.Streams}, {"its configuration (" + busConfInside + ")", theirs.Conf, ours.Conf}, {"its certificate (" + busCertificateInside + ")", theirs.Certificate, ours.Certificate}, {"its user list", theirs.Users, ours.Users}, } { if f.module == "" || f.module != f.bundle { differ = append(differ, fmt.Sprintf("%s: the module's is %q, the bundle's %q", f.what, f.module, f.bundle)) } } if len(differ) == 0 { return nil } return fmt.Errorf( "the bus the foundation raised is not where the %s module keeps it:\n - %s\n"+ "The module adopts the foundation's bus in place, so the bundle raises it with the module's "+ "name and paths; otherwise adopting it would raise a second bus beside the first, or an empty "+ "one on other paths (novox/hq ADR 0333 §1). Nothing has been registered", module, strings.Join(differ, "\n - ")) } func describeNames(names []string) string { if len(names) == 0 { return "no container at all" } return "the containers " + strings.Join(names, ", ") } // InstallVault installs the vault as a foundation module (novox/hq ADR 0085, amended). Nothing to // adopt and nothing to carry in: it is its own runtime, built from the catalogue like any provider, // and from its first push it keeps the export of every operator-sealed secret on its own disk. func InstallVault(ctx context.Context, o Options, control controlPlane, say func(string)) error { const module = "mesh-vault" manifest, err := readManifest(o.Catalogue, module) if err != nil { return err } if err := installProvider(ctx, o, control, module, manifest, "", nil, say); err != nil { return err } say(" installed " + module + " — the operator-sealed export now lives on this machine, outside the store") return nil } // storeIn finds the store container in the bundle this installer produced. func storeIn(d *declaration.Declaration) (*declaration.Container, error) { return foundationContainer(d, StoreID, "store") } // brokerIn finds the bus's container in the bundle this installer produced. func brokerIn(d *declaration.Declaration) (*declaration.Container, error) { return foundationContainer(d, BrokerID, "bus") } func foundationContainer(d *declaration.Declaration, id, what string) (*declaration.Container, error) { for _, r := range d.Resources { if r.Identity() != id { continue } container, ok := r.(*declaration.Container) if !ok { return nil, fmt.Errorf( "this bundle's %q is a %s, not a container, so the %s module has nothing to adopt", id, r.Kind(), what) } return container, nil } return nil, fmt.Errorf( "this bundle names no %q, so there is no %s for a module to adopt. It declares: %s", id, what, strings.Join(identities(d), ", ")) } // serverMatchesFoundation checks that the module's adopting container is the one the foundation // raised — same name, same image — so the applier reconciles it in place rather than replacing it. func serverMatchesFoundation(manifest []byte, store *declaration.Container, module string) error { var m struct { Resources []struct { Type string `json:"type"` Name string `json:"name"` Image string `json:"image"` } `json:"resources"` } if err := json.Unmarshal(manifest, &m); err != nil { return fmt.Errorf("the %s module's manifest is not readable: %w", module, err) } for _, r := range m.Resources { if r.Type != "container" || r.Name != store.Name { continue } if r.Image != store.Image { return fmt.Errorf( "the %s module's %q container is pinned to %q, and the foundation is running %q.\n"+ "The module adopts the foundation's store in place, so the two must name the same "+ "image — a different one would tear down the mesh's store and raise a new one on "+ "its data. Pin both to the same postgres image", module, store.Name, r.Image, store.Image) } return nil } return fmt.Errorf( "the %s module declares no container named %q, so it has nothing to adopt the foundation's "+ "store with. Its server container has to carry the name the foundation raised", module, store.Name) }