jochen
9e233e1d87
Never to be merged. The replay of hq issue 92 runs genesis at the commit before its fix, and the fix's own first parent stops at step 6 on issue 146; this is today's genesis without #91, so the bed reaches the push it replays (mesh-lab replays/register.go R92).
489 lines
20 KiB
Go
489 lines
20 KiB
Go
package bootstrap
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-host/internal/declaration"
|
|
)
|
|
|
|
// Phase three — nothing is special after installation (novox/hq issue 051).
|
|
//
|
|
// Genesis raises a store and a broker before any module system exists, because the control plane
|
|
// cannot ask provisioning for the store it keeps its own records in or the broker it is reached over
|
|
// (novox/hq ADR 0006). That leaves two servers behind: the foundation's, and a second one the
|
|
// `postgres` and bus modules used to raise for other modules to use. This turns the foundation's
|
|
// own servers into those modules, so a mesh runs ONE postgres and ONE bus — the control plane's
|
|
// contexts and the database of each module that asks for one, in the same server (WBS 3.1/3.2).
|
|
//
|
|
// **Adopted in place, not replaced.** The control plane is stateless and is swapped for a fresh
|
|
// container (control.go); the store and broker hold the mesh's memory and its bus, so they are kept.
|
|
// The module declares a container with the same name, image and spec the foundation raised, and the
|
|
// applier — which keys on the container name and compares a spec digest (mesh-host internal/apply) —
|
|
// finds it already running and leaves it be. The image is pinned to the one the foundation is
|
|
// running, read from the bundle this installer produced, so the two specs are the same digest and
|
|
// nothing is recreated. A recreate happens only on a real upgrade, which is where a stated window
|
|
// belongs (WBS 3.3).
|
|
|
|
// StoreID and BrokerID are what the foundation bundle calls the two servers it raises; the modules
|
|
// that adopt them are found by these ids in the bundle this installer produced, the same way the
|
|
// control plane's own container is (ControlPlaneID).
|
|
const (
|
|
StoreID = "store"
|
|
BrokerID = "broker"
|
|
)
|
|
|
|
// InstallStore makes the foundation's store the `postgres` module, adopted in place.
|
|
//
|
|
// The order is InstallFromCatalogue's, with two additions the store needs and an ordinary provider
|
|
// does not: the server image is pinned to the one the foundation is already running (so the module's
|
|
// container is the same spec and is adopted, not a second one raised), and the superuser password —
|
|
// the foundation's, made at genesis — is carried in through `secret accept`, because the mesh cannot
|
|
// invent a credential that already created the databases (the same reasoning as the control plane's
|
|
// store connections, control.go deliverStores).
|
|
func InstallStore(ctx context.Context, o Options, control controlPlane,
|
|
foundation *declaration.Declaration, say func(string)) error {
|
|
|
|
const module = "postgres"
|
|
manifest, err := readManifest(o.Catalogue, module)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
store, err := storeIn(foundation)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// The module adopts the running store rather than raising a second one, so its server container
|
|
// has to BE the foundation's — same name, same image. The applier keys on the name and compares
|
|
// a spec digest (internal/apply), so a mismatch here would not adopt the mesh's memory but
|
|
// replace it. Checked before anything is registered, so a drift between the two pinned upstream
|
|
// images (the catalogue's and the foundation bundle's) fails fast and by name, rather than
|
|
// surfacing as the mesh's store being torn down and recreated.
|
|
//
|
|
// Not rewritten to the foundation's: the server image travels through the builder, which reads
|
|
// the manifest from the repository and leaves a concrete image alone but would carry a rewrite
|
|
// nowhere. The two are kept equal at the source — one pinned postgres, named in both places.
|
|
if err := serverMatchesFoundation(manifest, store, module); err != nil {
|
|
return err
|
|
}
|
|
say(" adopting " + store.Name + " — the store the foundation raised, unchanged")
|
|
|
|
// The superuser is the foundation's, made at genesis — carried in before the push, or the push
|
|
// would seal random bytes where a working password has to be and the provisioner would not open
|
|
// the store it is meant to manage.
|
|
if err := installProvider(ctx, o, control, module, manifest, "(the provisioner; the server is adopted, not built)",
|
|
func() error {
|
|
return deliverCredential(ctx, o, control, module, "superuser", StoreSuperuserFile, "the store's superuser", say)
|
|
},
|
|
say); err != nil {
|
|
return err
|
|
}
|
|
say(" adopted mesh-store — the foundation's store is now the " + module + " module")
|
|
return nil
|
|
}
|
|
|
|
// installProvider registers, builds, issues, assigns and pushes one catalogue module, with one
|
|
// thing done just before the push — the moment a credential the mesh could not have made has to
|
|
// be in it. InstallFromCatalogue is the same sequence without that moment; the store, the broker
|
|
// and the vault each need it or need the shape, and three copies of it drifted.
|
|
func installProvider(ctx context.Context, o Options, control controlPlane, module string,
|
|
manifest []byte, buildNote string, beforePush func() error, say func(string)) error {
|
|
|
|
remote := "/" + module + "-module.json"
|
|
if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil {
|
|
return err
|
|
}
|
|
if _, err := control.tell(ctx, "module", "add", remote); err != nil {
|
|
return err
|
|
}
|
|
say(" registered " + module)
|
|
|
|
if o.CatalogSource.Repository == "" {
|
|
return fmt.Errorf("%s has to be built and there is no --catalog-source to build it from: "+
|
|
"the catalogue CHECKOUT says what it is, the catalogue REPOSITORY is where a builder "+
|
|
"clones it", module)
|
|
}
|
|
say(strings.TrimRight(" building "+module+" "+buildNote, " "))
|
|
if _, err := control.within(buildWait).tell(ctx, "build", o.CatalogSource.Repository,
|
|
"--path", "modules/"+module, "--ref", refOr(o.CatalogSource.Ref), "--wait", "1200s"); err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err := control.tell(ctx, "module", "issue", module, "--node", o.Node); err != nil {
|
|
say(" no account " + module + " — it declares nothing to say on the broker")
|
|
} else {
|
|
say(" account issued " + module)
|
|
}
|
|
|
|
if _, err := control.tell(ctx, "assign", o.Node, module); err != nil {
|
|
return err
|
|
}
|
|
if err := prepareModule(ctx, o, control, module, say); err != nil {
|
|
return err
|
|
}
|
|
if beforePush != nil {
|
|
if err := beforePush(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
_, err := pushNode(ctx, o, control, say)
|
|
return err
|
|
}
|
|
|
|
// deliverCredential carries a credential genesis made into a module as its own secret, through
|
|
// `secret accept`: the mesh cannot invent the value a running server already has.
|
|
func deliverCredential(ctx context.Context, o Options, control controlPlane, module, secret, file, what string,
|
|
say func(string)) error {
|
|
|
|
value, err := readCredentialFile(file)
|
|
if err != nil {
|
|
return fmt.Errorf("%s is not at %s, so the %s module has nothing to open its server with — "+
|
|
"and the mesh cannot invent the one the server already has: %w", what, file, module, err)
|
|
}
|
|
at := "/accepting-" + secret
|
|
if err := control.carryingSecret(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil {
|
|
return err
|
|
}
|
|
if _, err := control.tell(ctx, "secret", "accept", o.Node, module, secret, "--from", at); err != nil {
|
|
return err
|
|
}
|
|
say(" accepted " + secret + " — " + what + ", as genesis made it")
|
|
return nil
|
|
}
|
|
|
|
// BusModule is the module the foundation's bus is adopted as (novox/hq ADR 0331 §1). The previous
|
|
// broker's module, lavinmq, left the catalogue with the move to one bus (ADR 0131).
|
|
const BusModule = "nats"
|
|
|
|
// InstallBroker makes the foundation's bus the `nats` module, adopted in place — the same shape as
|
|
// InstallStore, for the same reasons: the bus holds the mesh's streams, so it is kept, and from the
|
|
// module's first push the bus's user list reaches it in the module's declaration like any module's
|
|
// (ADR 0331 §1, §5). The installer's own placing of that list (enrol.go placeTheBusUsers) was the
|
|
// bridge to here and is not run again.
|
|
//
|
|
// **The bundle raises the bus where the module keeps it** (novox/hq ADR 0333 §1): the same container
|
|
// name, the same streams directory, configuration directory, user list and certificate. That is checked
|
|
// here against the module's own manifest before anything is registered, and a difference is refused
|
|
// naming each one — under another name or on other paths the module would raise a second bus beside the
|
|
// first, or an empty one in its place.
|
|
//
|
|
// **The image is the one thing that changes, once** (ADR 0333 §3). The store's module pins the upstream
|
|
// postgres the foundation runs; the bus's module builds its server image (it carries the mesh's snapshot
|
|
// tool), so no pinned reference can be equal to the foundation's. The applier recreates the container on
|
|
// the module's image, on the same streams, users and certificate. This says so before the push and waits
|
|
// after it until the container runs the module's image and the bus answers, so nothing after this step
|
|
// dials a bus that is restarting.
|
|
func InstallBroker(ctx context.Context, o Options, control controlPlane,
|
|
foundation *declaration.Declaration, fetch func(context.Context, string) (int, string, error),
|
|
say func(string)) error {
|
|
|
|
const module = BusModule
|
|
manifest, err := readManifest(o.Catalogue, module)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
bus, err := brokerIn(foundation)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := busLayoutIsTheModules(manifest, foundation, module); err != nil {
|
|
return err
|
|
}
|
|
say(" adopting " + bus.Name + " — the bus the foundation raised, its streams kept")
|
|
if err := installProvider(ctx, o, control, module, manifest, "(the server, adopting the foundation's bus)",
|
|
func() error {
|
|
if running := runningImage(ctx, control, bus.Name); running != "" && running != bus.Image {
|
|
say(" already restarted " + bus.Name + " — it runs the module's image")
|
|
return nil
|
|
}
|
|
say(" restarting " + bus.Name + " — once, onto the " + module + " module's image; its " +
|
|
"streams, users and certificate stay where they are (novox/hq ADR 0333 §3)")
|
|
return nil
|
|
}, say); err != nil {
|
|
return err
|
|
}
|
|
if err := waitForTheAdoptedBus(ctx, o, control, bus, fetch, say); err != nil {
|
|
return err
|
|
}
|
|
say(" adopted " + bus.Name + " — the foundation's bus is now the " + module +
|
|
" module, and its user list is the module's from here on")
|
|
return nil
|
|
}
|
|
|
|
// busMonitor is where the bus says it is healthy: its monitoring port, published on the machine's own
|
|
// loopback by the bundle and by the module alike (both "127.0.0.1:8222:8222"). /healthz answers ok only
|
|
// once the server is up and JetStream has its streams back.
|
|
const busMonitor = "http://127.0.0.1:8222/healthz"
|
|
|
|
// runningImage is the image the named container was created from, or "" when there is none.
|
|
func runningImage(ctx context.Context, control controlPlane, name string) string {
|
|
asking, cancel := context.WithTimeout(ctx, control.timeout)
|
|
defer cancel()
|
|
out, err := control.run(asking, "docker", "inspect", "--format", "{{.Config.Image}}", name)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(out)
|
|
}
|
|
|
|
// waitForTheAdoptedBus waits until the bus container runs an image other than the one the bundle raised
|
|
// it from — the module's, after the applier's one recreate — and the bus answers on its monitor.
|
|
//
|
|
// Bounded by --wait, and asked at least once. A bus still on the bundle's image when it runs out is
|
|
// refused, naming what it runs: the push was sent and the node has not applied it, and every step after
|
|
// this one reaches the mesh through that bus.
|
|
func waitForTheAdoptedBus(ctx context.Context, o Options, control controlPlane, bus *declaration.Container,
|
|
fetch func(context.Context, string) (int, string, error), say func(string)) error {
|
|
|
|
deadline := time.Now().Add(o.Wait)
|
|
var running, said string
|
|
for {
|
|
running = runningImage(ctx, control, bus.Name)
|
|
if running != "" && running != bus.Image {
|
|
code, body, err := fetch(ctx, busMonitor)
|
|
if err == nil && code == http.StatusOK {
|
|
say(" bus answers " + bus.Name + " on " + running)
|
|
return nil
|
|
}
|
|
said = strings.TrimSpace(body)
|
|
if err != nil {
|
|
said = err.Error()
|
|
}
|
|
}
|
|
if !time.Now().Before(deadline) {
|
|
break
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-time.After(answerEvery):
|
|
}
|
|
}
|
|
switch {
|
|
case running == "":
|
|
return fmt.Errorf("the bus container %s is not there after the %s module was pushed, so nothing "+
|
|
"after this step could reach the mesh. The node-engine's journal says why it did not raise it",
|
|
bus.Name, BusModule)
|
|
case running == bus.Image:
|
|
return fmt.Errorf("%s still runs the bundle's image %s %s after the %s module was pushed: the node "+
|
|
"has not applied it. Its streams are untouched; run this installer again once the node-engine "+
|
|
"applies (its journal says why it did not)", bus.Name, bus.Image, o.Wait, BusModule)
|
|
}
|
|
return fmt.Errorf("%s runs the %s module's image %s and its monitor (%s) did not answer ok within %s: %s",
|
|
bus.Name, BusModule, running, busMonitor, o.Wait, orUnknown(said))
|
|
}
|
|
|
|
// BusUsersID is the bundle's resource for the bus's first user list.
|
|
const BusUsersID = "bus-accounts"
|
|
|
|
// busLayout is where a bus lives: its container, the machine's directories it mounts as its streams,
|
|
// its configuration and its certificate, and the file its user list is.
|
|
type busLayout struct {
|
|
Container, Streams, Conf, Certificate, Users string
|
|
}
|
|
|
|
// Where the server reads each, inside its container: the same in the bundle and the module, since both
|
|
// run the same server with the same configuration.
|
|
const (
|
|
busStreamsInside = "/data"
|
|
busConfInside = "/etc/nats"
|
|
busCertificateInside = "/tls"
|
|
)
|
|
|
|
// foundationBusLayout reads the layout of the bus the bundle raises.
|
|
func foundationBusLayout(foundation *declaration.Declaration) (busLayout, error) {
|
|
bus, err := brokerIn(foundation)
|
|
if err != nil {
|
|
return busLayout{}, err
|
|
}
|
|
at := mountedFrom(bus.Volumes)
|
|
l := busLayout{Container: bus.Name, Streams: at[busStreamsInside], Conf: at[busConfInside],
|
|
Certificate: at[busCertificateInside]}
|
|
for _, r := range foundation.Resources {
|
|
if f, ok := r.(*declaration.File); ok && r.Identity() == BusUsersID {
|
|
l.Users = f.Path
|
|
}
|
|
}
|
|
return l, nil
|
|
}
|
|
|
|
// moduleBusLayout reads the layout of the bus the module declares: its server container, the mounts of
|
|
// that container with every `${access:<id>}` resolved to the access's path, and its `bus-users`.
|
|
func moduleBusLayout(manifest []byte, module string) (busLayout, error) {
|
|
var m struct {
|
|
BusUsers string `json:"bus-users"`
|
|
Resources []struct {
|
|
ID string `json:"id"`
|
|
Type string `json:"type"`
|
|
Name string `json:"name"`
|
|
Volumes []string `json:"volumes"`
|
|
} `json:"resources"`
|
|
Accesses []struct {
|
|
ID string `json:"id"`
|
|
Path string `json:"path"`
|
|
} `json:"accesses"`
|
|
}
|
|
if err := json.Unmarshal(manifest, &m); err != nil {
|
|
return busLayout{}, fmt.Errorf("the %s module's manifest is not readable: %w", module, err)
|
|
}
|
|
var containers []int
|
|
for i, r := range m.Resources {
|
|
if r.Type == "container" {
|
|
containers = append(containers, i)
|
|
}
|
|
}
|
|
if len(containers) != 1 {
|
|
var names []string
|
|
for _, i := range containers {
|
|
names = append(names, m.Resources[i].Name)
|
|
}
|
|
return busLayout{}, fmt.Errorf("the %s module declares %s; the bus it adopts is one server, so it "+
|
|
"declares one container", module, describeNames(names))
|
|
}
|
|
server := m.Resources[containers[0]]
|
|
resolved := make([]string, 0, len(server.Volumes))
|
|
for _, v := range server.Volumes {
|
|
for _, a := range m.Accesses {
|
|
v = strings.ReplaceAll(v, "${access:"+a.ID+"}", a.Path)
|
|
}
|
|
resolved = append(resolved, v)
|
|
}
|
|
at := mountedFrom(resolved)
|
|
return busLayout{Container: server.Name, Streams: at[busStreamsInside], Conf: at[busConfInside],
|
|
Certificate: at[busCertificateInside], Users: m.BusUsers}, nil
|
|
}
|
|
|
|
// busLayoutIsTheModules checks that the bus the bundle raised is where the module keeps it, field by
|
|
// field, so the applier takes it over rather than raising a second bus beside it or an empty one in its
|
|
// place (novox/hq ADR 0333 §1). Every difference is named, with both sides.
|
|
func busLayoutIsTheModules(manifest []byte, foundation *declaration.Declaration, module string) error {
|
|
theirs, err := moduleBusLayout(manifest, module)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
ours, err := foundationBusLayout(foundation)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var differ []string
|
|
for _, f := range []struct{ what, module, bundle string }{
|
|
{"the server container", theirs.Container, ours.Container},
|
|
{"its streams (" + busStreamsInside + ")", theirs.Streams, ours.Streams},
|
|
{"its configuration (" + busConfInside + ")", theirs.Conf, ours.Conf},
|
|
{"its certificate (" + busCertificateInside + ")", theirs.Certificate, ours.Certificate},
|
|
{"its user list", theirs.Users, ours.Users},
|
|
} {
|
|
if f.module == "" || f.module != f.bundle {
|
|
differ = append(differ, fmt.Sprintf("%s: the module's is %q, the bundle's %q", f.what,
|
|
f.module, f.bundle))
|
|
}
|
|
}
|
|
if len(differ) == 0 {
|
|
return nil
|
|
}
|
|
return fmt.Errorf(
|
|
"the bus the foundation raised is not where the %s module keeps it:\n - %s\n"+
|
|
"The module adopts the foundation's bus in place, so the bundle raises it with the module's "+
|
|
"name and paths; otherwise adopting it would raise a second bus beside the first, or an empty "+
|
|
"one on other paths (novox/hq ADR 0333 §1). Nothing has been registered",
|
|
module, strings.Join(differ, "\n - "))
|
|
}
|
|
|
|
func describeNames(names []string) string {
|
|
if len(names) == 0 {
|
|
return "no container at all"
|
|
}
|
|
return "the containers " + strings.Join(names, ", ")
|
|
}
|
|
|
|
// InstallVault installs the vault as a foundation module (novox/hq ADR 0085, amended). Nothing to
|
|
// adopt and nothing to carry in: it is its own runtime, built from the catalogue like any provider,
|
|
// and from its first push it keeps the export of every operator-sealed secret on its own disk.
|
|
func InstallVault(ctx context.Context, o Options, control controlPlane, say func(string)) error {
|
|
const module = "mesh-vault"
|
|
manifest, err := readManifest(o.Catalogue, module)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := installProvider(ctx, o, control, module, manifest, "", nil, say); err != nil {
|
|
return err
|
|
}
|
|
say(" installed " + module + " — the operator-sealed export now lives on this machine, outside the store")
|
|
return nil
|
|
}
|
|
|
|
// storeIn finds the store container in the bundle this installer produced.
|
|
func storeIn(d *declaration.Declaration) (*declaration.Container, error) {
|
|
return foundationContainer(d, StoreID, "store")
|
|
}
|
|
|
|
// brokerIn finds the bus's container in the bundle this installer produced.
|
|
func brokerIn(d *declaration.Declaration) (*declaration.Container, error) {
|
|
return foundationContainer(d, BrokerID, "bus")
|
|
}
|
|
|
|
func foundationContainer(d *declaration.Declaration, id, what string) (*declaration.Container, error) {
|
|
for _, r := range d.Resources {
|
|
if r.Identity() != id {
|
|
continue
|
|
}
|
|
container, ok := r.(*declaration.Container)
|
|
if !ok {
|
|
return nil, fmt.Errorf(
|
|
"this bundle's %q is a %s, not a container, so the %s module has nothing to adopt",
|
|
id, r.Kind(), what)
|
|
}
|
|
return container, nil
|
|
}
|
|
return nil, fmt.Errorf(
|
|
"this bundle names no %q, so there is no %s for a module to adopt. It declares: %s",
|
|
id, what, strings.Join(identities(d), ", "))
|
|
}
|
|
|
|
// serverMatchesFoundation checks that the module's adopting container is the one the foundation
|
|
// raised — same name, same image — so the applier reconciles it in place rather than replacing it.
|
|
func serverMatchesFoundation(manifest []byte, store *declaration.Container, module string) error {
|
|
var m struct {
|
|
Resources []struct {
|
|
Type string `json:"type"`
|
|
Name string `json:"name"`
|
|
Image string `json:"image"`
|
|
} `json:"resources"`
|
|
}
|
|
if err := json.Unmarshal(manifest, &m); err != nil {
|
|
return fmt.Errorf("the %s module's manifest is not readable: %w", module, err)
|
|
}
|
|
for _, r := range m.Resources {
|
|
if r.Type != "container" || r.Name != store.Name {
|
|
continue
|
|
}
|
|
if r.Image != store.Image {
|
|
return fmt.Errorf(
|
|
"the %s module's %q container is pinned to %q, and the foundation is running %q.\n"+
|
|
"The module adopts the foundation's store in place, so the two must name the same "+
|
|
"image — a different one would tear down the mesh's store and raise a new one on "+
|
|
"its data. Pin both to the same postgres image",
|
|
module, store.Name, r.Image, store.Image)
|
|
}
|
|
return nil
|
|
}
|
|
return fmt.Errorf(
|
|
"the %s module declares no container named %q, so it has nothing to adopt the foundation's "+
|
|
"store with. Its server container has to carry the name the foundation raised",
|
|
module, store.Name)
|
|
}
|
|
|
|
// orUnknown is s, or a word for nothing: trust.go held it, and this branch takes trust.go out.
|
|
func orUnknown(s string) string {
|
|
if strings.TrimSpace(s) == "" {
|
|
return "(nothing)"
|
|
}
|
|
return s
|
|
}
|