diff --git a/cmd/mesh-bootstrap/main.go b/cmd/mesh-bootstrap/main.go index 894462c..633f668 100644 --- a/cmd/mesh-bootstrap/main.go +++ b/cmd/mesh-bootstrap/main.go @@ -8,8 +8,11 @@ // cannot be folded into it without making that sentence false. Same tier, same repository, // different program. // -// What it does not do is enrol this machine, register modules or assign them. It stops at a -// running substrate with a control plane that answers, which is a mesh of one node. +// Genesis is a pivot (novox/hq ADR 0067). It raises a substrate whose control plane is named by the +// digest of its own configuration — legal exactly where nothing could have served an image — then +// enrols this machine, installs the registry module, pushes that image into it to get the manifest +// digest it has never had, reinstalls the control plane as an ordinary module pinned to it, and +// drops the temporary one. Without --catalog it stops after the substrate and says why. package main import ( @@ -17,7 +20,9 @@ import ( "encoding/json" "flag" "fmt" + "io" "net" + "net/http" "os" "os/signal" "syscall" @@ -35,27 +40,56 @@ var version = "development build" const ( defaultTemplate = "substrate.lock" defaultOut = "/var/lib/mesh-host/substrate.lock" + defaultRegistry = "127.0.0.1:5000" + defaultHost = "/usr/local/bin/mesh-host" + defaultService = "mesh-host.service" ) const usage = `mesh-bootstrap — make a bare machine into a mesh - bootstrap preflight, load, bundle, apply, verify (the default) + bootstrap the ten steps below (the default) version + 1 preflight what has to be true before anything is changed + 2 load the control plane's image, carried in this installer + 3 bundle the substrate, named for this machine + 4 apply raise it + 5 verify it is up, and the control plane replies + 6 enrol this machine becomes the mesh's first node + 7 registry install the module that gives this mesh an image store + 8 publish push the control plane's image into it, for its first digest + 9 control reinstall the control plane as an ordinary module, pinned to that digest + 10 retire drop the temporary control plane; the host removes it + --bundle the substrate template to build this machine's bundle from (default ` + defaultTemplate + `) --out where the produced bundle is written, for a person to read (default ` + defaultOut + `) --state where this node records what it has applied (default ` + store.DefaultPath + `) + --catalog a checkout of the mesh's catalogue, holding the registry's and the + control plane's manifests. Without it this stops after step 5 + --node the name this machine is known by (default: its hostname) + --registry where this mesh keeps its own images (default ` + defaultRegistry + `) + every node pulls the control plane from this, so on a mesh of more + than one machine it must be an address the others can reach + --host the mesh-host binary on this machine (default ` + defaultHost + `) + --host-service the unit that supervises it (default ` + defaultService + `) + --host-in-background start the host unsupervised instead. It does not survive + a reboot. This is what a lab does and what no real machine should --system which operating system this is; by default it is asked --timeout how long any single probe may take (default 30s) --wait how long a thing that is merely starting is given (default 3m) --dry-run everything that does not change the machine --json machine-readable output -It carries the control plane's image and applies a substrate. Every step is idempotent: -run it again after fixing whatever it named, and the steps that already succeeded say so. +Genesis is a pivot: a temporary control plane installs the registry that makes it +permanent. The temporary one is called temp-mesh-control and the permanent one is +called mesh-control, so they are two containers with two owners and there is nothing +to hand over. + +Every step is idempotent: run it again after fixing whatever it named, and the steps +that already succeeded say so. ` func main() { @@ -85,6 +119,14 @@ func parseArgs(args []string) (string, bootstrap.Options, bool, error) { Template: defaultTemplate, Out: defaultOut, State: store.DefaultPath, + Registry: defaultRegistry, + Host: defaultHost, + // The machine's own name, because that is what a person already calls it and an installer + // inventing a different one would leave the mesh naming a machine nobody recognises. It is + // read here rather than inside the bootstrap so that --node overrides a fact rather than a + // default computed halfway through. + Node: hostname(), + HostService: defaultService, // Longer than the host's 10s: these probes reach a container runtime that may be busy // pulling, and a probe that times out on a working machine is a false refusal. Timeout: 30 * time.Second, @@ -131,6 +173,14 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet { set.StringVar(&opts.Template, "bundle", opts.Template, "the substrate template to build from") set.StringVar(&opts.Out, "out", opts.Out, "where the produced bundle is written") set.StringVar(&opts.State, "state", opts.State, "where this node records what it has applied") + set.StringVar(&opts.Catalogue, "catalog", opts.Catalogue, + "a checkout of the mesh's catalogue; without it this stops after the substrate") + set.StringVar(&opts.Node, "node", opts.Node, "the name this machine is known by") + set.StringVar(&opts.Registry, "registry", opts.Registry, "where this mesh keeps its own images") + set.StringVar(&opts.Host, "host", opts.Host, "the mesh-host binary on this machine") + set.StringVar(&opts.HostService, "host-service", opts.HostService, "the unit that supervises it") + set.BoolVar(&opts.HostInBackground, "host-in-background", false, + "start the host unsupervised; it does not survive a reboot") set.StringVar(&opts.System, "system", opts.System, "which operating system this is") set.DurationVar(&opts.Timeout, "timeout", opts.Timeout, "how long any single probe may take") set.DurationVar(&opts.Wait, "wait", opts.Wait, "how long something merely starting is given") @@ -148,8 +198,9 @@ func run(ctx context.Context, command string, opts bootstrap.Options, jsonOut bo } } result, err := bootstrap.Run(ctx, opts, bootstrap.Deps{ - Run: apply.ExecRunner, - Dial: dial, + Run: apply.ExecRunner, + Dial: dial, + Fetch: fetch, }, say) // Printed whichever way it went. What the installer got through before it stopped is on @@ -177,6 +228,46 @@ func run(ctx context.Context, command string, opts bootstrap.Options, jsonOut bo } } +// hostname is what this machine calls itself, or empty. +// +// Empty rather than a guess: a machine that cannot say its own name is one the installer must be +// told about, and `mesh-bootstrap-0` would be a name in the mesh's records that matches nothing +// anybody types anywhere else. The refusal happens at step 6, where the name is first needed. +func hostname() string { + name, err := os.Hostname() + if err != nil { + return "" + } + return name +} + +// fetch asks an HTTP endpoint and reports what it said. +// +// Plain HTTP, and only at the mesh's own registry: it is reached over the mesh's private network, +// which is already the encrypted and authenticated thing, and a second layer inside it would be +// certificates to issue and rotate for no property the first does not have (mesh-control's +// `internal/builder` pushes to it on the same reasoning). +// +// The body is read with a limit. What is asked for is a status and a short JSON answer, and a +// registry that answered with a gigabyte would otherwise be an installer that never returns. +func fetch(ctx context.Context, url string) (int, string, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return 0, "", err + } + response, err := http.DefaultClient.Do(request) + if err != nil { + return 0, "", err + } + defer response.Body.Close() + + said, err := io.ReadAll(io.LimitReader(response.Body, 1<<20)) + if err != nil { + return response.StatusCode, "", err + } + return response.StatusCode, string(said), nil +} + // dial answers whether a TCP address responds. // // A connection rather than a ping or a name lookup: what has to work is a pull, and a pull opens a diff --git a/cmd/mesh-bootstrap/main_test.go b/cmd/mesh-bootstrap/main_test.go index 6731428..95815d1 100644 --- a/cmd/mesh-bootstrap/main_test.go +++ b/cmd/mesh-bootstrap/main_test.go @@ -105,9 +105,60 @@ func TestTheUsageTextAndTheFlagsAgree(t *testing.T) { t.Errorf("--%s exists and the usage text does not mention it", name) } } - for _, promised := range []string{"bundle", "out", "state", "system", "timeout", "wait", "dry-run", "json"} { + for _, promised := range []string{ + "bundle", "out", "state", "system", "timeout", "wait", "dry-run", "json", + "catalog", "node", "registry", "host", "host-service", "host-in-background", + } { if !declared[promised] { t.Errorf("the usage text promises --%s and no such flag exists", promised) } } } + +// The pivot's defaults are the documented ones too, and the one that has no default is the one +// that decides whether the pivot happens at all. +func TestThePivotsDefaultsAreTheDocumentedOnes(t *testing.T) { + _, opts, _, err := parseArgs(nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if opts.Registry != defaultRegistry { + t.Errorf("default registry is %q; the usage text says %q", opts.Registry, defaultRegistry) + } + if opts.Host != defaultHost { + t.Errorf("default host binary is %q; the usage text says %q", opts.Host, defaultHost) + } + if opts.HostService != defaultService { + t.Errorf("default host service is %q; the usage text says %q", + opts.HostService, defaultService) + } + if opts.HostInBackground { + t.Error("the host is started unsupervised by default, and no real machine should") + } + // **No default, deliberately.** A catalogue this installer went looking for on its own would + // be a checkout somebody else made, at whatever commit they left it on — and it decides which + // image the mesh's control plane is pinned to for ever after. + if opts.Catalogue != "" { + t.Errorf("--catalog defaults to %q; without one the installer stops at the substrate", + opts.Catalogue) + } + // The machine's own name, because that is what a person already calls it. + if opts.Node == "" { + t.Error("no default node name; this machine can say what it is called") + } +} + +// --node overrides the machine's own name rather than being ignored because a default was already +// computed. The name is what the mesh's records, tokens, assignments and pushes all name. +func TestTheNodeNameCanBeSaid(t *testing.T) { + _, opts, _, err := parseArgs([]string{"--node", "anchor", "--catalog", "/somewhere"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if opts.Node != "anchor" { + t.Errorf("--node anchor parsed as %q", opts.Node) + } + if opts.Catalogue != "/somewhere" { + t.Errorf("--catalog parsed as %q", opts.Catalogue) + } +} diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 44163b1..1771aee 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -25,6 +25,7 @@ package bootstrap import ( "context" "fmt" + "strings" "time" ) @@ -33,15 +34,28 @@ import ( type Step string const ( - StepPreflight Step = "preflight" - StepLoad Step = "load" - StepBundle Step = "bundle" - StepApply Step = "apply" - StepVerify Step = "verify" + StepPreflight Step = "preflight" + StepLoad Step = "load" + StepBundle Step = "bundle" + StepApply Step = "apply" + StepVerify Step = "verify" + StepEnrol Step = "enrol" + StepRegistry Step = "registry" + StepPublish Step = "publish" + StepControlPlane Step = "control-plane" + StepRetire Step = "retire" ) -// Steps in the order they happen, so a failure can say "step 2 of 5". -var Steps = []Step{StepPreflight, StepLoad, StepBundle, StepApply, StepVerify} +// Steps in the order they happen, so a failure can say "step 2 of 10". +// +// The first five make a machine; the last five make a mesh that can maintain itself. They are one +// program because they are one procedure — the whole reason the pivot exists is that steps 7 to 9 +// cannot happen without steps 1 to 5, and steps 1 to 5 leave something that cannot be upgraded +// without steps 7 to 9 (novox/hq ADR 0067). +var Steps = []Step{ + StepPreflight, StepLoad, StepBundle, StepApply, StepVerify, + StepEnrol, StepRegistry, StepPublish, StepControlPlane, StepRetire, +} // Error is a failure, named by the step it happened in. type Error struct { @@ -86,8 +100,37 @@ type Options struct { // Wait is how long something that is merely starting is given: a socket-activated container // runtime, a control plane opening its stores. Wait time.Duration + + // Node is the name this machine is known by in the mesh. Everything after the substrate names + // it: the record, the token, the assignment, the push. + Node string + + // Catalogue is a checkout of the mesh's catalogue repository, which is where the registry's and + // the control plane's manifests are read from. Empty stops the installer after the substrate: + // there is no pivot without manifests, and pretending otherwise would leave a machine that + // looks installed and cannot upgrade itself. + Catalogue string + + // Registry is where this mesh's own images live, as this machine reaches it. Every node will + // pull the control plane from what this says, so on a mesh of more than one machine it must be + // an address the others can reach. + Registry string + + // Host is the `mesh-host` binary on this machine — the program that enrols and then holds the + // machine to what the mesh says. The installer runs it; it does not contain it. + Host string + // HostService is the unit that supervises it. Started and enabled, never written: what a unit + // says is a packaging decision, and an installer inventing one would put a file on the machine + // that whatever installed the host will disagree with. + HostService string + // HostInBackground starts the host unsupervised instead, which is what the lab does and what no + // real machine should do — it does not survive a reboot. + HostInBackground bool } +// pivots reports whether this run goes past the substrate. +func (o Options) pivots() bool { return strings.TrimSpace(o.Catalogue) != "" } + // Deps are the ways this program reaches outside itself. Injected so the whole of it can be // tested without a container runtime, a network, or a machine to break — the same reason // `internal/apply` takes a Runner (novox/hq ADR 0017). @@ -97,6 +140,10 @@ type Deps struct { // Dial reports whether a TCP address answers, for "can this machine reach the registries the // bundle names". Dial func(ctx context.Context, address string) error + // Fetch asks an HTTP endpoint and reports what it said. Used only against the mesh's own + // registry: a container that is up is not a registry that serves, and `/v2/` is the one + // question whose answer means it is. + Fetch func(ctx context.Context, url string) (int, string, error) } // Result is what the bootstrap did, in the shape `--json` prints. @@ -123,10 +170,34 @@ type Result struct { // Running is the substrate's containers, confirmed up. Running []string `json:"running,omitempty"` - // Answered is what the control plane said back — not merely that it is up. - Answered string `json:"control-plane,omitempty"` + // Answered is what the temporary control plane said back — not merely that it is up. + Answered string `json:"temporary-control-plane,omitempty"` + // Temporary is what the substrate's control plane is called, which is not what the module's is. + Temporary string `json:"temporary-container,omitempty"` - // Stopped names why a dry run went no further. Empty on a real run. + // Node is this machine's name in the mesh, and how it came to be enrolled and heard from. + Node string `json:"node,omitempty"` + NodeAdded bool `json:"node-record-created,omitempty"` + Enrolled bool `json:"enrolled-now,omitempty"` + Agent string `json:"host-agent,omitempty"` + + // Registry is the mesh's own artifact store, once it answers. + Registry string `json:"registry,omitempty"` + RegistryReplied int `json:"registry-replied,omitempty"` + RegistryKnown bool `json:"registry-already-registered,omitempty"` + RegistryRunning string `json:"registry-container,omitempty"` + PublishedAs string `json:"control-plane-image,omitempty"` + PublishedAlready bool `json:"control-plane-image-already-published,omitempty"` + + // Permanent is the control plane as an ordinary module. + Permanent string `json:"permanent-container,omitempty"` + PermanentAnswered string `json:"permanent-control-plane,omitempty"` + StoresDelivered []string `json:"stores-delivered,omitempty"` + TemporaryRetired bool `json:"temporary-retired,omitempty"` + TemporaryWasGone bool `json:"temporary-was-already-gone,omitempty"` + TemporaryRemovedAt int `json:"resources-removed,omitempty"` + + // Stopped names why a run went no further. Empty on a run that pivoted. Stopped string `json:"stopped,omitempty"` } @@ -141,9 +212,41 @@ type Result struct { // answer to it is to run this again: re-running is the retry, and it is one a person chooses after // reading which step failed and why. // -// **What this does NOT do: enrolment, the module catalogue, and assignment.** It stops at a running -// substrate with a control plane that replies — a mesh of one node with nothing joined to it. See -// the marker at the end. +// **Genesis is a pivot** (novox/hq ADR 0067). Steps 1 to 5 raise a substrate whose control plane is +// named by the digest of its own configuration, because nothing has ever served that image and +// nothing could have. Steps 6 to 10 turn that into a mesh that can maintain itself: this machine +// enrols, the registry module is installed, the carried image is pushed INTO that registry — which +// gives it a manifest digest, its first — and the control plane is reinstalled as an ordinary +// module pinned to it. The temporary one is then dropped from the bundle and the host removes it. +// +// **What makes the last part expressible is a name.** The substrate's control plane is called +// `temp-mesh-control` and the module's is called `mesh-control`. Two containers, two owners: +// nothing is handed over, nothing has to stop being owned without being destroyed, and destruction +// by omission is the right end for something named "temp". +// +// Without --catalog it stops after step 5 and says so, because there are no manifests to install +// and a machine that looks installed and cannot upgrade itself is worse than one that stopped. +// +// **What an interruption leaves, at every step, and how a re-run continues.** This matters more +// here than anywhere else in the repository, because a machine left without a control plane cannot +// be fixed remotely — so no step may leave one: +// +// 1–3 nothing on the machine but a written file. Re-run: the bundle is produced again. +// 4 a partly-raised substrate, recorded in the state file. Re-run: apply converges the rest. +// 5 everything up; something did not answer yet. Re-run: it is asked again. +// 6 a node record and possibly a spent token. Re-run: `node list` finds the record, the +// identity file says whether this machine enrolled, and a fresh token is issued if not. +// 7 the registry registered, assigned, maybe not applied. Re-run: registered again (an +// upsert), pushed again, waited for again. The temporary control plane is untouched. +// 8 the image pushed and the digest unread. Re-run: the registry is asked what it holds and +// the answer is the same digest; nothing is pushed twice. +// 9 the module registered and the container not yet up, OR up beside the temporary one. Both +// are working states: the mesh has a control plane throughout. Re-run: continues. +// 10 the bundle rewritten and the container still there. Re-run: the bundle already omits it +// and the apply removes it; a bundle that already omits it reports "already dropped". +// +// The only step that cannot be undone by re-running is enrolment, and that is refused rather than +// repeated: a second identity is one the mesh does not know, and the mesh believes the first. func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, error) { if say == nil { say = func(string) {} @@ -181,12 +284,21 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, failed(StepBundle, err) } result.BundleWas, result.BundlePlaces, result.Bundle = rewritten.Was, rewritten.Places, o.Out + result.Temporary = rewritten.TempName + if rewritten.Renamed { + say(fmt.Sprintf(" control plane %s, renamed from %s", + rewritten.TempName, rewritten.WasCalled)) + say(" the permanent one is a module and takes the plain name; " + + "this one is dropped at the end") + } else { + say(" control plane " + rewritten.TempName + " — the template already named it that") + } if rewritten.Changed { - say(fmt.Sprintf(" control plane %s", rewritten.Now)) + say(fmt.Sprintf(" its image %s", rewritten.Now)) say(fmt.Sprintf(" replacing %s, named in %d place(s)", rewritten.Was, rewritten.Places)) } else { - say(fmt.Sprintf(" control plane %s — the template already named it, nothing rewritten", + say(fmt.Sprintf(" its image %s — the template already named it, nothing rewritten", rewritten.Now)) } for _, kept := range rewritten.Kept { @@ -207,6 +319,15 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro // loading, applying and asking the result questions, and none of those can be answered by // not doing them. say(fmt.Sprintf(" would write %s (%d resources)", o.Out, rewritten.Resources)) + if o.pivots() { + // Named rather than attempted. Everything from step 6 on is a conversation with a + // control plane that a dry run has not raised, so there is nothing to ask and nothing + // honest to report about the answers. + say(" would then enrol " + o.Node + ", install the registry from " + + o.Catalogue + ", push the control plane's image into it,") + say(" reinstall the control plane as a module, and drop " + + rewritten.TempName) + } result.Stopped = "dry run: the bundle was produced and checked, and nothing was written, " + "loaded or applied" say("\n" + result.Stopped) @@ -242,19 +363,75 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, failed(StepVerify, err) } - say("\nthis machine is a mesh of one node, with nothing joined to it yet.") + if !o.pivots() { + // Stopped, and said plainly. What has been raised works and cannot be upgraded: its + // control plane is named by an image id, which no registry serves, so nothing can ever + // replace it with a newer one. That is the whole of what the pivot fixes, and it needs + // manifests, and manifests come from a checkout somebody has to point this at. + result.Stopped = "no --catalog was given, so this stopped at the substrate. " + + "The control plane is named by the digest of its own configuration and no registry " + + "serves it, so this mesh cannot yet upgrade itself. Run again with " + + "--catalog to finish the pivot; every step " + + "above will say it is already done" + say("\nthis machine is a mesh of one node, with nothing joined to it yet.") + say(result.Stopped) + return result, nil + } - // NEXT STAGE — NOT IMPLEMENTED HERE. + // ---- 6. enrol ------------------------------------------------------------------------- // - // What remains between "a mesh exists" and "a mesh does something": issuing this machine a - // token and enrolling it as its own first node, registering the module catalogue with the - // control plane, and assigning modules to nodes. All three are conversations with the control - // plane that has just been proved to reply, so they belong after this point and inside none of - // the steps above. - // - // Left out rather than half-written. Everything above changes a machine; all of that changes a - // mesh, and a program that did both would have two jobs and one name. - say("not done here: enrolment, the module catalogue, and assignment.") + // From here on the mesh is being told things, and the way to tell it anything is to run its + // own binary inside its own container. `temporary` is the substrate's control plane; the + // module's is a different container with a different name and does not exist yet. + temporary := controlPlane{container: rewritten.TempName, run: d.Run, timeout: o.Timeout} + + say("enrol — this machine joins the mesh it is running") + enrolled, err := Enrol(ctx, o, sys, temporary, say) + result.Node, result.NodeAdded, result.Enrolled = enrolled.Node, enrolled.Added, enrolled.Joined + result.Agent = enrolled.Agent + if err != nil { + return result, failed(StepEnrol, err) + } + + // ---- 7. registry ---------------------------------------------------------------------- + say("registry — somewhere for this mesh to keep its own images") + registry, err := InstallRegistry(ctx, o, d, temporary, say) + result.Registry, result.RegistryRunning = registry.Address, registry.Container + result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered + if err != nil { + return result, failed(StepRegistry, err) + } + + // ---- 8. publish ----------------------------------------------------------------------- + say("publish — the control plane's image gets its first manifest digest") + published, err := PublishControlPlane(ctx, o, d, loaded.ID, say) + result.PublishedAs, result.PublishedAlready = published.Reference, published.Already + if err != nil { + return result, failed(StepPublish, err) + } + + // ---- 9. control plane ----------------------------------------------------------------- + say("control plane — installed as an ordinary module, pinned to that digest") + permanent, err := InstallControlPlane(ctx, o, d, temporary, rewritten.Declaration, + published.Reference, say) + result.Permanent, result.PermanentAnswered = permanent.Container, permanent.Answered + result.StoresDelivered = permanent.Delivered + if err != nil { + return result, failed(StepControlPlane, err) + } + + // ---- 10. retire ----------------------------------------------------------------------- + say("retire — the temporary control plane is dropped from the bundle") + retired, err := RetireTheTemporaryControlPlane(ctx, o, sys, rewritten.Bundle, d.Run, say) + result.TemporaryRetired = retired.Gone || retired.Already + result.TemporaryWasGone, result.TemporaryRemovedAt = retired.Already, retired.Removed + if err != nil { + return result, failed(StepRetire, err) + } + + say("\nthis machine is a mesh of one node, and the control plane it runs is a module " + + "pinned to an image its own registry serves.") + say("what remains is somebody else's: adding nodes, and assigning what they should run.") return result, nil } diff --git a/internal/bootstrap/control.go b/internal/bootstrap/control.go new file mode 100644 index 0000000..c406b53 --- /dev/null +++ b/internal/bootstrap/control.go @@ -0,0 +1,335 @@ +package bootstrap + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/novox/mesh-host/internal/declaration" +) + +// storeFileSuffix is how a manifest asks for a store connection in a file rather than in the +// environment. +// +// `MESH_STORE_` is what the control plane reads (mesh-control's `internal/store`.Variable) +// and putting a password in a container's environment puts it in `docker inspect` for ever. So a +// module manifest names a file per context and points at it with `…_FILE`; the mesh seals the value +// into that file on the machine, and nothing but the process reads it. +const storeFileSuffix = "_FILE" + +// storeVariablePrefix is the front of the same names. +const storeVariablePrefix = "MESH_STORE_" + +// Permanent is what step 9 did. +type Permanent struct { + Installed + // Container is what the module calls its container, confirmed running. + Container string + // Image is what it is pinned to — the digest step 8's push produced. + Image string + // Delivered is every store connection accepted into it, by secret name. + Delivered []string + // Answered is what the permanent control plane said back. + Answered string +} + +// InstallControlPlane makes the control plane an ordinary module. +// +// **The host performs the replacement, not the control plane** (novox/hq ADR 0067). The temporary +// control plane composes a declaration naming the registry-pinned image, publishes it, and this +// node's host creates the container. Nothing is asked to replace itself while running, which is +// what makes the whole thing expressible: the container being created is called `mesh-control` and +// the one composing it is called `temp-mesh-control`, so there are two of them and neither is in +// the other's way. +// +// **The store connections are the substrate's, made at genesis, and the mesh cannot invent them.** +// Every other secret in a mesh is one the mesh made; these existed before the mesh did — they are +// the credentials the substrate bundle created the databases with. Generating replacements would +// put thirty-two random bytes where a working connection string has to be, and the control plane +// would come up unable to open a single context. So they go in through `secret accept`, which is +// exactly the path for a value the mesh must carry and could not have invented — and they are read +// out of the bundle this installer produced rather than reconstructed, because the bundle is what +// created them and a second opinion about what a DSN should say is a second chance to be wrong. +func InstallControlPlane(ctx context.Context, o Options, d Deps, control controlPlane, + substrate *declaration.Declaration, image string, say func(string)) (Permanent, error) { + + out := Permanent{Image: image} + + manifest, err := readManifest(o.Catalogue, ControlPlaneModule) + if err != nil { + return out, fmt.Errorf( + "%w\n"+ + "This is the manifest that makes the control plane an ordinary module. Without it "+ + "the machine keeps the temporary control plane the substrate raised, which works "+ + "and cannot be upgraded — so the install stops here rather than pretending to "+ + "have pivoted", err) + } + + pinned, places, err := pinImage(manifest, image) + if err != nil { + return out, err + } + say(fmt.Sprintf(" pinned %s, in %d place(s)", image, places)) + + container, _, err := containerIn(pinned, controlPlaneResourceIn(pinned)) + if err != nil { + return out, err + } + out.Container = container + if container == "" { + return out, fmt.Errorf( + "the %s module's container has no name, so nothing can be verified afterwards", + ControlPlaneModule) + } + + installed, err := registerAndAssign(ctx, o, control, ControlPlaneModule, pinned, say) + out.Installed = installed + if err != nil { + return out, err + } + + // The connections, before the push that would otherwise deliver random bytes for them. + delivered, err := deliverStores(ctx, o, control, pinned, substrate, say) + out.Delivered = delivered + if err != nil { + return out, err + } + + if out.Pushed, err = pushNode(ctx, o, control, say); err != nil { + return out, err + } + + if err := waitForContainer(ctx, control.run, o.Timeout, o.Wait, container, say); err != nil { + return out, err + } + // And it answers, which is the same question step 5 asked of the temporary one and for the + // same reason: `status` opens all three stores, so a reply proves the sealed connections it + // was given are the ones the substrate made. Asked of the NEW container — this is the only + // moment in the program where two control planes are running, and asking the wrong one would + // report the temporary one's health as the permanent one's. + answered, err := waitForTheControlPlane(ctx, control.run, o.Timeout, o.Wait, container, say) + if err != nil { + return out, err + } + out.Answered = answered + return out, nil +} + +// pinImage replaces the catalogue's placeholder digest with what the registry assigned. +// +// **Textual, and every place it appears.** A manifest may name its image in more than one resource +// — the catalogue's converted modules routinely carry a runtime container beside the application's +// — and the same reasoning as the bundle rewrite applies: replacing one and not the others leaves +// something pointing at an image nothing serves, and it fails half way through an apply rather +// than here. +// +// It refuses a manifest with no placeholder in it. That is not pedantry: a manifest already +// carrying a real digest is one somebody pinned by hand, and quietly registering it would install a +// control plane that is not the image this machine just published — which is the one thing this +// step exists to guarantee. +func pinImage(manifest []byte, reference string) ([]byte, int, error) { + places := bytes.Count(manifest, []byte(placeholderDigest)) + if places == 0 { + return nil, 0, fmt.Errorf( + "the %s module's manifest carries no placeholder digest (%s), so there is nothing to "+ + "pin to the image this machine just published.\n"+ + "A manifest already naming a digest was pinned by somebody else, to some other "+ + "build. Registering it would install a control plane that is not the one this "+ + "installer carried and pushed", ControlPlaneModule, placeholderDigest) + } + // The reference the registry gave back is `/@sha256:…`, and what the + // manifest holds is `@sha256:0…0`. Replacing only the digest would leave the + // manifest's own repository name in front of it — which may be `mesh-control` with no + // registry, and a runtime would then pull it from the internet. The whole reference moves. + var out bytes.Buffer + rest := manifest + for { + at := bytes.Index(rest, []byte(placeholderDigest)) + if at < 0 { + out.Write(rest) + break + } + // Back up over the repository this digest belongs to, which runs to the opening quote. + start := bytes.LastIndexByte(rest[:at], '"') + if start < 0 { + return nil, 0, fmt.Errorf( + "the %s module's manifest has a placeholder digest that is not inside a JSON "+ + "string, so the installer cannot tell what image it belongs to", ControlPlaneModule) + } + out.Write(rest[:start+1]) + out.WriteString(reference) + rest = rest[at+len(placeholderDigest):] + } + pinned := out.Bytes() + + // Read back. A substitution on text can catch more than it was aimed at, and the manifest is + // about to be handed to the mesh as the description of what it runs. + var checked map[string]any + if err := json.Unmarshal(pinned, &checked); err != nil { + return nil, 0, fmt.Errorf( + "pinning the %s module's image broke its manifest: %w", ControlPlaneModule, err) + } + if bytes.Contains(pinned, []byte(placeholderDigest)) { + return nil, 0, fmt.Errorf( + "the %s module's manifest still carries a placeholder digest after pinning", + ControlPlaneModule) + } + return pinned, places, nil +} + +// controlPlaneResourceIn is the id of the resource that runs the control plane. +// +// The manifest is written by the catalogue and the installer does not get to name its resources. +// What it can do is find the one container whose image is the one just pinned — and when a manifest +// declares exactly one container, that is the answer without any searching at all. +func controlPlaneResourceIn(manifest []byte) string { + var m struct { + Resources []struct { + ID string `json:"id"` + Type string `json:"type"` + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + return "" + } + var containers []string + for _, r := range m.Resources { + if r.Type == "container" { + containers = append(containers, r.ID) + } + } + if len(containers) == 1 { + return containers[0] + } + // More than one, so the name has to be guessed at rather than derived — and the catalogue's + // own convention for the resource that IS the module is `container`, with anything else beside + // it named for what it does. + for _, id := range containers { + if id == "container" || id == ControlPlaneModule || id == "control-plane" { + return id + } + } + return "" +} + +// deliverStores carries the substrate's own database connections into the module. +// +// The pairing is read from the manifest rather than assumed, so that whatever the catalogue calls +// these secrets is what is delivered: a container asking for `MESH_STORE_INVENTORY_FILE` names a +// path, and the module's own-secret that writes that path is the secret to accept the connection +// as. That is one lookup and it cannot get the wrong secret — the alternative, guessing that the +// secret is called `inventory`, would seal a connection string under a name nothing reads and +// leave the mesh to invent random bytes for the one that is. +func deliverStores(ctx context.Context, o Options, control controlPlane, manifest []byte, + substrate *declaration.Declaration, say func(string)) ([]string, error) { + + wanted, err := storeSecretsIn(manifest) + if err != nil { + return nil, err + } + if len(wanted) == 0 { + return nil, fmt.Errorf( + "the %s module's manifest asks for no store connections. A control plane reaches each "+ + "context through its own credential (novox/hq ADR 0008), so a manifest naming none "+ + "describes a control plane that can open nothing.\n"+ + "The shape this installer delivers into is a file per context, named by an "+ + "own-secret, with %s%s in the container's environment pointing at it", + ControlPlaneModule, storeVariablePrefix, storeFileSuffix) + } + + temporary, err := controlPlaneIn(substrate) + if err != nil { + return nil, err + } + + var delivered []string + for _, context := range sortedKeys(wanted) { + secret := wanted[context] + connection := temporary.Env[storeVariablePrefix+context] + if strings.TrimSpace(connection) == "" { + return delivered, fmt.Errorf( + "the %s module wants the %s store's connection and the bundle this installer "+ + "produced does not name one: its control plane has no %s.\n"+ + "These connections are the substrate's, created at genesis — the mesh cannot "+ + "invent them and the installer will not guess at one", + ControlPlaneModule, strings.ToLower(context), storeVariablePrefix+context) + } + + // Into the container as a file, because `secret accept` reads a file or a prompt and the + // installer has neither a terminal to be prompted at nor a way to write to a command's + // standard input through the runner every applier in this repository shares. + at := "/accepting-" + strings.ToLower(context) + if err := control.carrying(ctx, "mesh-store-"+strings.ToLower(context), + []byte(connection), at); err != nil { + return delivered, err + } + if _, err := control.tell(ctx, "secret", "accept", o.Node, ControlPlaneModule, secret, + "--from", at); err != nil { + return delivered, err + } + delivered = append(delivered, secret) + say(" accepted " + secret + " — the " + strings.ToLower(context) + + " store, as the substrate made it") + } + return delivered, nil +} + +// storeSecretsIn pairs each context with the secret its connection must be accepted as. +// +// Read out of the manifest twice over: the container's environment says which contexts are wanted +// and what file each expects, and the module's own-secrets say which secret writes which file. A +// pair that does not meet is refused rather than half-delivered. +func storeSecretsIn(manifest []byte) (map[string]string, error) { + var m struct { + OwnSecrets map[string]string `json:"own-secrets"` + Resources []struct { + Type string `json:"type"` + Env map[string]string `json:"env"` + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + return nil, fmt.Errorf("the %s module's manifest is not readable: %w", ControlPlaneModule, err) + } + + byPath := map[string]string{} + for name, path := range m.OwnSecrets { + byPath[path] = name + } + + wanted := map[string]string{} + for _, r := range m.Resources { + if r.Type != "container" { + continue + } + for key, path := range r.Env { + if !strings.HasPrefix(key, storeVariablePrefix) || !strings.HasSuffix(key, storeFileSuffix) { + continue + } + context := strings.TrimSuffix(strings.TrimPrefix(key, storeVariablePrefix), storeFileSuffix) + secret, ok := byPath[path] + if !ok { + return nil, fmt.Errorf( + "the %s module's container reads the %s store's connection from %s, and no "+ + "own-secret of that module writes that file.\n"+ + "So the mesh would seal nothing there and the control plane would find an "+ + "empty file where a connection string has to be. The manifest has to name "+ + "the two ends the same", + ControlPlaneModule, strings.ToLower(context), path) + } + wanted[context] = secret + } + } + return wanted, nil +} + +func sortedKeys(m map[string]string) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/internal/bootstrap/control_test.go b/internal/bootstrap/control_test.go new file mode 100644 index 0000000..bf4d97c --- /dev/null +++ b/internal/bootstrap/control_test.go @@ -0,0 +1,213 @@ +package bootstrap + +import ( + "context" + "strings" + "testing" + "time" +) + +// Step 9 is where the control plane stops being a special case. These tests defend the two things +// that could go wrong quietly: pinning it to the wrong image, and delivering it store connections +// the mesh invented rather than the ones the substrate actually made. + +// theControlPlaneModule is the shape this installer codes against: one container using the +// catalogue's placeholder-digest convention, with each store connection delivered as a sealed +// secret written into a file and MESH_STORE__FILE pointing at it. +const theControlPlaneModule = `{ + "module": "mesh-control", + "version": "1", + "capabilities": ["container-runtime"], + "own-secrets": { + "inventory-store": "/var/lib/mesh/control/inventory", + "identity-store": "/var/lib/mesh/control/identity", + "licences-store": "/var/lib/mesh/control/licences" + }, + "resources": [ + {"id": "state", "type": "directory", "path": "/var/lib/mesh/control", "mode": "0700"}, + {"id": "container", "type": "container", "name": "mesh-control", + "image": "mesh-control@` + placeholderDigest + `", + "network": "host", "args": ["serve"], + "env": { + "MESH_STORE_INVENTORY_FILE": "/var/lib/mesh/control/inventory", + "MESH_STORE_IDENTITY_FILE": "/var/lib/mesh/control/identity", + "MESH_STORE_LICENCES_FILE": "/var/lib/mesh/control/licences" + }} + ] +}` + +const pushedReference = "127.0.0.1:5000/mesh-control@sha256:" + + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + +// **The whole reference moves, not only the digest.** The manifest's placeholder names a +// repository too, and replacing sixty-four zeros inside it would leave `mesh-control@sha256:…` +// with no registry in front — which a runtime would go to the internet for, and this mesh's +// control plane exists in no public registry by design. +func TestTheControlPlaneIsPinnedToWhatThisMeshsRegistryAssigned(t *testing.T) { + pinned, places, err := pinImage([]byte(theControlPlaneModule), pushedReference) + if err != nil { + t.Fatal(err) + } + if places != 1 { + t.Errorf("the placeholder was found in %d place(s)", places) + } + if !strings.Contains(string(pinned), `"image": "`+pushedReference+`"`) { + t.Errorf("the manifest does not name the pushed image:\n%s", pinned) + } + if strings.Contains(string(pinned), `"mesh-control@sha256:`) { + t.Errorf("the digest was replaced and the manifest's own repository name was left in "+ + "front of it, so nothing says which registry serves it:\n%s", pinned) + } +} + +// A manifest already naming a real digest was pinned by somebody else, to some other build. +// Registering it would install a control plane that is not the image this machine just published, +// which is the one thing this step exists to guarantee. +func TestAManifestAlreadyPinnedByHandIsRefused(t *testing.T) { + already := strings.Replace(theControlPlaneModule, placeholderDigest, + "sha256:"+strings.Repeat("9", 64), 1) + if _, _, err := pinImage([]byte(already), pushedReference); err == nil { + t.Fatal("a manifest already pinned to some other image was accepted") + } +} + +// Every placeholder moves. A manifest naming its image in a second resource — a runtime container +// beside the application's, which the catalogue's converted modules routinely carry — would +// otherwise be left half pinned, and fail inside an apply rather than here. +func TestEveryPlaceTheManifestNamesTheImageIsPinned(t *testing.T) { + twice := strings.Replace(theControlPlaneModule, + `{"id": "state", "type": "directory", "path": "/var/lib/mesh/control", "mode": "0700"},`, + `{"id": "state", "type": "directory", "path": "/var/lib/mesh/control", "mode": "0700"}, + {"id": "migrate", "type": "container", "name": "mesh-control-migrate", "run-once": true, + "image": "mesh-control@`+placeholderDigest+`", "args": ["migrate"]},`, 1) + + pinned, places, err := pinImage([]byte(twice), pushedReference) + if err != nil { + t.Fatal(err) + } + if places != 2 { + t.Errorf("the placeholder was found in %d place(s), and the manifest names it twice", places) + } + if strings.Contains(string(pinned), placeholderDigest) { + t.Error("a placeholder survived the pinning") + } +} + +// **The connections are the substrate's, and they are read out of the bundle that made them.** +// The mesh cannot invent them: they are the credentials the substrate created the databases with, +// and thirty-two random bytes in their place would leave the control plane unable to open a single +// context. The pairing is read from the manifest so that whatever the catalogue calls these +// secrets is what is delivered. +func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) { + wanted, err := storeSecretsIn([]byte(theControlPlaneModule)) + if err != nil { + t.Fatal(err) + } + for context, secret := range map[string]string{ + "INVENTORY": "inventory-store", + "IDENTITY": "identity-store", + "LICENCES": "licences-store", + } { + if wanted[context] != secret { + t.Errorf("the %s store's connection would be accepted as %q, want %q", + context, wanted[context], secret) + } + } + + // And the values are the substrate's own, taken from the produced bundle rather than composed. + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + runtime := &asked{answer: aMeshThatAgrees(nil)} + control := controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second} + + delivered, err := deliverStores(context.Background(), Options{Node: "anchor"}, control, + []byte(theControlPlaneModule), rewritten.Declaration, func(string) {}) + if err != nil { + t.Fatal(err) + } + if len(delivered) != 3 { + t.Fatalf("%d connections were delivered, and the mesh holds three contexts: %v", + len(delivered), delivered) + } + for _, secret := range delivered { + if !runtime.ran("secret accept anchor mesh-control " + secret + " --from") { + t.Errorf("%s was not accepted through `secret accept`: %v", secret, runtime.commands) + } + } +} + +// A manifest whose container reads a file no own-secret writes is refused. The mesh would seal +// nothing there and the control plane would find an empty file where a connection string has to +// be — which presents as a control plane that will not start, three steps from the cause. +func TestAConnectionFileNothingWritesIsRefused(t *testing.T) { + mismatched := strings.Replace(theControlPlaneModule, + `"inventory-store": "/var/lib/mesh/control/inventory"`, + `"inventory-store": "/var/lib/mesh/control/somewhere-else"`, 1) + + _, err := storeSecretsIn([]byte(mismatched)) + if err == nil { + t.Fatal("a manifest whose two ends do not meet was accepted") + } + if !strings.Contains(err.Error(), "own-secret") { + t.Errorf("the refusal does not say which half is missing: %v", err) + } +} + +// A manifest asking for no store connections at all describes a control plane that can open +// nothing, and the refusal says what shape the installer delivers into — because the manifest is +// written in another repository and this is where the two have to agree. +func TestAManifestWantingNoStoresIsRefusedWithTheShapeItShouldHave(t *testing.T) { + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + runtime := &asked{answer: aMeshThatAgrees(nil)} + control := controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second} + + bare := `{"module":"mesh-control","version":"1","resources":[ + {"id":"container","type":"container","name":"mesh-control", + "image":"mesh-control@` + placeholderDigest + `"}]}` + + _, err = deliverStores(context.Background(), Options{Node: "anchor"}, control, + []byte(bare), rewritten.Declaration, func(string) {}) + if err == nil { + t.Fatal("a control plane that can open nothing was accepted") + } + if !strings.Contains(err.Error(), storeVariablePrefix+""+storeFileSuffix) { + t.Errorf("the refusal does not say what shape is expected: %v", err) + } +} + +// The permanent control plane is asked a question, not merely looked at — the same question the +// temporary one was asked at step 5, and for the same reason: `status` opens all three stores, so +// a reply proves the sealed connections it was given are the ones the substrate made. +func TestThePermanentControlPlaneIsAskedTheSameQuestion(t *testing.T) { + runtime := &asked{answer: aMeshThatAgrees(map[string]string{ + "module list": "", + "exec mesh-control /mesh-control": "1 node, 0 waiting\n", + })} + control := controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second} + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + + out, err := InstallControlPlane(context.Background(), + installing(t, catalogueWith(t, ControlPlaneModule, theControlPlaneModule)), + Deps{Run: runtime.run}, control, rewritten.Declaration, pushedReference, func(string) {}) + if err != nil { + t.Fatal(err) + } + if out.Answered != "1 node, 0 waiting" { + t.Errorf("the permanent control plane's reply is reported as %q", out.Answered) + } + if !runtime.ran("docker exec mesh-control " + controlPlaneBinary + " status") { + t.Errorf("the permanent control plane was never asked anything: %v", runtime.commands) + } + // And the module was registered with the digest, not with the placeholder. + if !runtime.ran("module add /mesh-control-module.json") { + t.Errorf("the module was never registered: %v", runtime.commands) + } +} diff --git a/internal/bootstrap/enrol.go b/internal/bootstrap/enrol.go new file mode 100644 index 0000000..2387ec3 --- /dev/null +++ b/internal/bootstrap/enrol.go @@ -0,0 +1,276 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/system" +) + +// hereIs what `node list` says about a machine the mesh has heard from recently. +// +// mesh-control prints one of three words per node: "here", "never spoken", or "out of touch ". +// The installer waits for the first, and it is the only honest proof that the host agent is +// running: an enrolled machine whose host is not running looks exactly like an enrolled machine +// whose host has crashed, and both look exactly like a successful install until the first push +// silently applies nothing. +const hereIs = "here" + +// Enrolled is what step 6 did. +type Enrolled struct { + // Node is the name this machine is known by. + Node string + // Added is true when the mesh had no record and one was created. + Added bool + // Joined is true when this machine enrolled during this run. False on a re-run. + Joined bool + // Agent says how the host came to be running: what was found, or what was started. + Agent string +} + +// Enrol makes this machine the mesh's first node, and gets the host agent running on it. +// +// **The mesh is running and nothing has joined it.** Steps 1 to 5 leave a store, a broker and a +// control plane that answers — a mesh of one node in the sense that it has one machine and zero +// node records. Everything after this point is the control plane being *told* things, and none of +// it reaches a machine until a host is running there to hear it. +// +// Four things, in this order, each asked before it is done: +// +// 1. a node record, unless `node list` already shows one +// 2. a one-time token, unless this machine already holds an identity +// 3. `mesh-host enrol`, which is the machine presenting the identity it already had +// 4. the host agent running, and the mesh saying it has heard from it +// +// **Step 3 is the one that cannot be undone by re-running.** A machine that has enrolled holds an +// identity the mesh has recorded, and enrolling again would replace it with a second one the mesh +// does not know — `mesh-host enrol` refuses exactly this, and so does the check here, one layer +// earlier and with a sentence about what to do. +// +// `control.run` is this machine's runner and not only the control plane's: the same injected +// Runner reaches the container, the service manager and the host binary, which is what lets the +// whole of this be tested without any of the three. +func Enrol(ctx context.Context, o Options, sys system.System, control controlPlane, + say func(string)) (Enrolled, error) { + + out := Enrolled{Node: o.Node} + if strings.TrimSpace(o.Node) == "" { + return out, errors.New( + "this machine has no name to be known by. Give one with --node; it is what the mesh " + + "records, what a token is issued against, and what every later assignment names") + } + + // 1. The record. + nodes, err := control.tell(ctx, "node", "list") + if err != nil { + return out, err + } + if mentions(nodes, o.Node) { + say(" already a node " + o.Node) + } else { + if _, err := control.tell(ctx, "node", "add", o.Node); err != nil { + return out, err + } + out.Added = true + say(" node record " + o.Node) + } + + // 2 and 3. The identity, and being known. + // + // Asked of this machine's own identity file rather than of the mesh, because the two answer + // different questions: the mesh knows whether a record exists, and only the machine knows + // whether it holds the key that record names. + where := identity.Path(o.State) + switch mine, err := identity.Load(where); { + case err == nil && mine.Node == o.Node: + say(" already enrolled " + o.Node + " — " + where) + case err == nil: + return out, fmt.Errorf( + "this machine is already node %q and was asked to become %q.\n"+ + "Re-enrolling replaces the identity the mesh has recorded, so it is not something "+ + "an installer does on its own. Run with --node %s, or remove %s and start over "+ + "deliberately", mine.Node, o.Node, mine.Node, where) + case !errors.Is(err, identity.ErrNoIdentity): + return out, fmt.Errorf("cannot read this machine's identity at %s: %w", where, err) + default: + said, err := control.tell(ctx, "token", "issue", "--node", o.Node) + if err != nil { + return out, err + } + token, err := tokenIn(said) + if err != nil { + return out, err + } + joining, cancel := context.WithTimeout(ctx, o.Wait) + joined, err := control.run(joining, o.Host, "enrol", "--token", token, "--state", o.State) + cancel() + if err != nil { + return out, fmt.Errorf( + "%s would not enrol this machine: %w\n%s\n"+ + "The token is one-time and has now been spent; running this again issues "+ + "another, so a re-run is safe", o.Host, err, indent(strings.TrimSpace(joined))) + } + if !strings.Contains(joined, "enrolled as "+o.Node) { + // Exit zero and no such sentence. Refused rather than believed: the host says exactly + // this line on success, and something that succeeded without saying it did something + // else. + return out, fmt.Errorf( + "%s exited happily and did not say it enrolled as %s:\n%s", + o.Host, o.Node, indent(strings.TrimSpace(joined))) + } + out.Joined = true + say(" enrolled as " + o.Node) + } + + // 4. The agent. + agent, err := runTheHost(ctx, o, sys, control, say) + if err != nil { + return out, err + } + out.Agent = agent + return out, nil +} + +// runTheHost makes sure something on this machine is listening to the mesh, and proves it. +// +// **The installer does not install the service, and says so.** A unit file is a packaging decision +// — where the binary lives, which user it runs as, what it is called — and an installer that +// invented one would be putting a file on the machine that whatever installed `mesh-host` will +// later disagree with. So this starts a unit that is already there and refuses when there is none. +// +// It goes through the host's OWN system abstraction rather than running systemctl itself, for the +// reason `ApplyBundle` gives about applying: two implementations of "is this service running" is +// how the installer and the host come to disagree about a machine. It also gets the right refusal +// for free — `ServiceState` treats a unit that does not exist as an error and never as "stopped", +// which is exactly the distinction that matters here. +// +// --host-in-background is the lab's arrangement, kept because the lab is what exercises this and +// it has no service. It is loud about what it is, because a host started this way is gone at the +// next reboot and the mesh would go quiet for reasons nobody would connect to an install. +func runTheHost(ctx context.Context, o Options, sys system.System, control controlPlane, + say func(string)) (string, error) { + + // Asked first, and of the mesh rather than of the process table. What matters is not that a + // process exists but that the mesh has heard from it, and only one of those two is the thing + // every later step depends on. + if heard, err := heardFrom(ctx, control, o.Node); err != nil { + return "", err + } else if heard { + say(" host running the mesh has heard from " + o.Node) + return "already running", nil + } + + how := "" + switch { + case o.HostInBackground: + // Detached, and not waited for: this process runs until the machine stops, so a runner + // that captures output would never return. `nohup … &` through a shell is how the lab + // starts it and is deliberately the same command. + started, cancel := context.WithTimeout(ctx, o.Timeout) + _, err := control.run(started, "sh", "-c", + fmt.Sprintf("nohup %s run --state %s >> /var/log/mesh-host.log 2>&1 &", o.Host, o.State)) + cancel() + if err != nil { + return "", fmt.Errorf("cannot start %s in the background: %w", o.Host, err) + } + how = "started in the background" + say(" host running started in the background — THIS DOES NOT SURVIVE A REBOOT.") + say(" A real machine needs " + o.HostService + " installed and enabled.") + + default: + state, err := sys.ServiceState(ctx, control.run, o.HostService) + if err != nil { + return "", fmt.Errorf( + "the mesh has not heard from %s and this machine has no %s to start: %w\n"+ + "The host has to be running for anything the mesh says to reach this machine. "+ + "Install the service that supervises it and run this again — every step "+ + "before this one will say it is already done. In a lab, --host-in-background "+ + "starts it unsupervised instead, and that is not an install", + o.Node, o.HostService, err) + } + if state != "running" { + if err := sys.SetServiceState(ctx, control.run, o.HostService, "running"); err != nil { + return "", fmt.Errorf("cannot start %s: %w", o.HostService, err) + } + how = "started " + o.HostService + say(" host running started " + o.HostService) + } else { + // Running, and the mesh has not heard from it. Not an error yet — it may have started + // a second ago — so it is waited for below like everything else that is merely + // starting. + how = o.HostService + " was already running" + say(" host running " + o.HostService + " is running") + } + // At boot as well, or the machine comes back without a mesh and nothing says why. + if boot, err := sys.ServiceBoot(ctx, control.run, o.HostService); err == nil && boot != "enabled" { + if err := sys.SetServiceBoot(ctx, control.run, o.HostService, "enabled"); err != nil { + return "", fmt.Errorf("cannot make %s start at boot: %w", o.HostService, err) + } + say(" host at boot " + o.HostService + " enabled") + } + } + + // Read back (novox/hq ADR 0018). A service that started and a mesh that has heard from a node + // are two different facts, and only the second is the one every later step rests on. + deadline := time.Now().Add(o.Wait) + for { + heard, err := heardFrom(ctx, control, o.Node) + if err != nil { + return "", err + } + if heard { + say(" the mesh hears " + o.Node) + return how, nil + } + if time.Now().After(deadline) { + return "", fmt.Errorf( + "%s is running and the mesh has not heard from %s after %s.\n"+ + "The host links to the broker at the address the token carried — if that "+ + "address is not one this machine can reach, this is where it shows. Read the "+ + "host's own output, and check MESH_BROKER_ADDRESS in the bundle this "+ + "installer wrote", o.HostService, o.Node, o.Wait) + } + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(answerEvery): + } + } +} + +// heardFrom asks the mesh whether this node has spoken to it. +func heardFrom(ctx context.Context, control controlPlane, node string) (bool, error) { + listing, err := control.tell(ctx, "node", "list") + if err != nil { + return false, err + } + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == node { + return fields[1] == hereIs, nil + } + } + return false, nil +} + +// tokenIn finds the token in what `token issue` said. +// +// The same rule the lab uses: the one long unbroken line. `token issue` prints an explanation +// around it, and a token is a signed blob with no spaces in it, so "long and unbroken" identifies +// it without this having to know the format — which is what keeps the installer from having an +// opinion about a thing the control plane owns. +func tokenIn(said string) (string, error) { + for _, line := range strings.Split(said, "\n") { + line = strings.TrimSpace(line) + if len(line) > 100 && !strings.ContainsAny(line, " \t") { + return line, nil + } + } + return "", fmt.Errorf( + "the mesh issued a token and there is no token in what it said:\n%s", + indent(strings.TrimSpace(said))) +} diff --git a/internal/bootstrap/enrol_test.go b/internal/bootstrap/enrol_test.go new file mode 100644 index 0000000..7561d56 --- /dev/null +++ b/internal/bootstrap/enrol_test.go @@ -0,0 +1,223 @@ +package bootstrap + +import ( + "context" + "crypto/ed25519" + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/system" +) + +// Step 6 is where the mesh stops being something running on a machine and starts being something +// the machine belongs to. What these tests defend is that it cannot happen twice, and that +// "installed" is never claimed for a machine the mesh has not actually heard from. + +// alreadyEnrolled writes an identity file, as `mesh-host enrol` leaves behind. +func alreadyEnrolled(t *testing.T, node string) string { + t.Helper() + state := filepath.Join(t.TempDir(), "state.json") + mine, err := identity.Generate(node) + if err != nil { + t.Fatal(err) + } + // A whole membership, because an identity that cannot reach its mesh is refused on the way in + // — which is the right refusal and not the one being tested here. + signer, _, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + mine.Membership = identity.Membership{ + Broker: "192.0.2.10:5671", Fingerprint: "sha256:whatever", + Signer: signer, Password: "issued-at-enrolment", + } + if err := identity.Save(identity.Path(state), mine); err != nil { + t.Fatal(err) + } + return state +} + +func arch(t *testing.T) system.System { + t.Helper() + chosen, err := system.For("arch") + if err != nil { + t.Fatal(err) + } + return chosen +} + +// A machine that has already enrolled is not enrolled again, and no token is spent on it. The +// installer is run over and over; a second identity is one the mesh does not know, and the mesh +// believes the first. +func TestAMachineThatHasAlreadyEnrolledIsNotEnrolledAgain(t *testing.T) { + runtime := &asked{answer: func(name string, args []string) (string, error) { + joined := strings.Join(args, " ") + switch { + case strings.Contains(joined, "node list"): + return "anchor here 01J0\n", nil + } + return "", fmt.Errorf("unexpected: %s %v", name, args) + }} + + out, err := Enrol(context.Background(), Options{ + Node: "anchor", State: alreadyEnrolled(t, "anchor"), Timeout: time.Second, + }, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err != nil { + t.Fatal(err) + } + if out.Joined || out.Added { + t.Error("a machine that had already enrolled enrolled again") + } + if runtime.ran("token issue") { + t.Errorf("a token was issued for a machine that already holds an identity: %v", + runtime.commands) + } + if out.Agent != "already running" { + t.Errorf("the host agent is reported as %q", out.Agent) + } +} + +// A machine already enrolled under ANOTHER name is refused, with what to do about it. Re-enrolling +// replaces the identity the mesh recorded, which is a deliberate act and not something an +// installer does on its own. +func TestAMachineEnrolledUnderAnotherNameIsRefused(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if strings.Contains(strings.Join(args, " "), "node list") { + return "somewhere-else here 01J0\n", nil + } + return "", nil + }} + + _, err := Enrol(context.Background(), Options{ + Node: "anchor", State: alreadyEnrolled(t, "somewhere-else"), Timeout: time.Second, + }, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err == nil { + t.Fatal("a machine already enrolled as something else was enrolled again") + } + for _, wanted := range []string{"somewhere-else", "--node"} { + if !strings.Contains(err.Error(), wanted) { + t.Errorf("the refusal does not mention %q:\n%v", wanted, err) + } + } +} + +// **The mesh having heard from the node is the proof, not a process existing.** A host that is +// running and cannot reach the broker looks exactly like a successful install until the first push +// silently applies nothing — which is the class of fault this whole program exists to stop being +// found late. +func TestAHostThatIsRunningAndUnheardOfIsNotAnInstall(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + runtime := &asked{answer: func(_ string, args []string) (string, error) { + joined := strings.Join(args, " ") + switch { + case strings.Contains(joined, "node list"): + // Enrolled, and never spoken. + return "anchor never spoken 01J0\n", nil + case args[0] == "show": + return "LoadState=loaded\nActiveState=active\n", nil + case args[0] == "is-enabled": + return "enabled\n", nil + } + return "", nil + }} + + _, err := Enrol(context.Background(), Options{ + Node: "anchor", State: alreadyEnrolled(t, "anchor"), HostService: "mesh-host.service", + Timeout: time.Second, Wait: 0, + }, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err == nil { + t.Fatal("a node the mesh has never heard from was reported enrolled and running") + } + if !strings.Contains(err.Error(), "MESH_BROKER_ADDRESS") { + t.Errorf("the failure does not name the thing that is silently fatal when wrong:\n%v", err) + } +} + +// A machine with no service to start is refused, and the refusal says what is missing rather than +// inventing a unit file. What a unit says is a packaging decision, and an installer writing one +// would put a file on the machine that whatever installed the host will disagree with. +func TestAMachineWithNoHostServiceIsRefusedRatherThanGivenOne(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + joined := strings.Join(args, " ") + switch { + case strings.Contains(joined, "node list"): + return "anchor never spoken 01J0\n", nil + case args[0] == "show": + // systemd knows nothing about it. + return "LoadState=not-found\nActiveState=inactive\n", nil + } + return "", nil + }} + + _, err := Enrol(context.Background(), Options{ + Node: "anchor", State: alreadyEnrolled(t, "anchor"), HostService: "mesh-host.service", + Timeout: time.Second, Wait: 0, + }, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err == nil { + t.Fatal("a machine with no host service was reported as having a running host") + } + for _, wanted := range []string{"mesh-host.service", "--host-in-background"} { + if !strings.Contains(err.Error(), wanted) { + t.Errorf("the refusal does not mention %q:\n%v", wanted, err) + } + } +} + +// A machine with no name is refused before anything is said to the mesh. The name is what the +// record, the token, the assignment and the push all name, and one the installer invented would +// match nothing anybody types anywhere else. +func TestAMachineWithNoNameIsRefusedBeforeAnythingIsAsked(t *testing.T) { + runtime := &asked{answer: func(string, []string) (string, error) { + return "", fmt.Errorf("nothing should have been asked") + }} + _, err := Enrol(context.Background(), Options{Timeout: time.Second}, arch(t), + controlPlane{container: "temp-mesh-control", run: runtime.run}, func(string) {}) + if err == nil { + t.Fatal("a machine with no name was enrolled") + } + if len(runtime.commands) != 0 { + t.Errorf("the mesh was asked something first: %v", runtime.commands) + } +} + +// The token is found in what the mesh said, by the rule the lab uses: the one long unbroken line. +// The installer does not parse a format the control plane owns. +func TestTheTokenIsFoundInWhatTheMeshSaid(t *testing.T) { + said := "a token for anchor, good once:\n\n " + strings.Repeat("t", 240) + "\n\n" + + "carry it to the machine and run: mesh-host enrol --token \n" + token, err := tokenIn(said) + if err != nil { + t.Fatal(err) + } + if token != strings.Repeat("t", 240) { + t.Errorf("the token was read as %q", token) + } + + if _, err := tokenIn("nothing here that looks like one\n"); err == nil { + t.Fatal("an answer with no token in it was accepted") + } +} + +// A listing's name is matched as a whole word at the start of a line, so `registry` is not found +// inside `registry-mirror`. A substring match would report a module installed that is not, and the +// installer would skip creating it. +func TestAListingIsMatchedByNameAndNotBySubstring(t *testing.T) { + listing := "registry-mirror 1 built abc\nother 1 built def\n" + if mentions(listing, "registry") { + t.Error("registry-mirror was read as registry") + } + if !mentions(listing, "registry-mirror") { + t.Error("registry-mirror was not found") + } +} diff --git a/internal/bootstrap/module.go b/internal/bootstrap/module.go new file mode 100644 index 0000000..2c6eb43 --- /dev/null +++ b/internal/bootstrap/module.go @@ -0,0 +1,145 @@ +package bootstrap + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" +) + +// placeholderDigest is what the catalogue writes where a built image's digest will go. +// +// Sixty-four zeros. A manifest in the catalogue names an image the mesh builds and pushes, and +// until that has happened there is no digest to name — so the convention is a digest that is +// obviously not one, replaced by the pipeline when it publishes. The installer meets it twice: it +// must NOT be there in the registry's manifest, whose image is upstream and never built +// (novox/hq 04-ISSUES/029), and it MUST be there in the control plane's, which is the image +// step 8 has just pushed. +const placeholderDigest = "sha256:" + "0000000000000000000000000000000000000000000000000000000000000000" + +// catalogueDir is where a mesh-catalog checkout keeps its manifests. +const catalogueDir = "modules" + +// manifestFile is the path a module's manifest is read from, given a catalogue checkout. +func manifestFile(catalogue, module string) string { + return filepath.Join(catalogue, catalogueDir, module, "module.json") +} + +// readManifest reads one module's manifest out of a mesh-catalog checkout. +// +// **From a checkout rather than from anything the mesh serves**, and that is the ordering the whole +// pivot exists to respect: at this point the mesh has no build machine, no forge and — until step 7 +// finishes — no registry. What a manifest is, is a file; the installer is handed the directory it +// is in and reads it. +func readManifest(catalogue, module string) ([]byte, error) { + path := manifestFile(catalogue, module) + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf( + "the %s module's manifest could not be read: %w\n"+ + "--catalog is a checkout of the mesh's catalogue repository, and this is read from "+ + "%s inside it", module, err, filepath.Join(catalogueDir, module, "module.json")) + } + return raw, nil +} + +// Installed is what installing one module did. +type Installed struct { + // Module is its name. + Module string + // Known is true when the mesh already had this module in its catalogue. The manifest is + // registered either way — a re-run with a changed manifest must land — so this reports what + // was found rather than what was skipped. + Known bool + // Assigned is true when this node was given the module during this run. + Assigned bool + // Pushed is what the mesh said when it sent this node its declaration. + Pushed string +} + +// installModule registers a manifest, gives it to this node, and sends it. +// +// The three verbs a person types, in the order they type them, through the same commands. There is +// no installer-only path into the mesh: everything here is `module add`, `assign` and `push`, so +// what the installer does on a bare machine and what an operator does on a running mesh are the +// same act (novox/hq ADR 0035, one refusal per act however it is asked for). +func installModule(ctx context.Context, o Options, control controlPlane, module string, + manifest []byte, say func(string)) (Installed, error) { + + out, err := registerAndAssign(ctx, o, control, module, manifest, say) + if err != nil { + return out, err + } + out.Pushed, err = pushNode(ctx, o, control, say) + return out, err +} + +// registerAndAssign is the half of installing that happens before anything is sent. +// +// Split out because one module needs something in between: the control plane's own store +// connections have to be accepted before its declaration is composed, or the mesh would seal +// thirty-two random bytes into the file it expects a connection string in and the container would +// come up unable to open anything (mesh-control's `secret accept`, and what it exists for). +// +// **`module add` is run every time and is not skipped when the module is already known.** It is an +// upsert on the manifest, and the manifest is exactly what changes between runs — step 9 registers +// the control plane with a digest that did not exist the first time. Skipping it because the name +// was already in the catalogue would silently pin the mesh to the previous image. +func registerAndAssign(ctx context.Context, o Options, control controlPlane, module string, + manifest []byte, say func(string)) (Installed, error) { + + out := Installed{Module: module} + + known, err := control.tell(ctx, "module", "list") + if err != nil { + return out, err + } + out.Known = mentions(known, module) + + remote := "/" + module + "-module.json" + if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil { + return out, err + } + if _, err := control.tell(ctx, "module", "add", remote); err != nil { + return out, err + } + if out.Known { + say(" registered " + module + " — the mesh already knew it; the manifest is now this one") + } else { + say(" registered " + module) + } + + assigned, err := control.tell(ctx, "assign", o.Node, module) + if err != nil { + return out, err + } + out.Assigned = true + say(" assigned " + module + " to " + o.Node) + if refusal := strings.TrimSpace(assigned); strings.Contains(refusal, "but ") { + // `assign` records what a person meant and says at once when the machine cannot host it. + // Repeated rather than swallowed: the push below will apply everything else and this is + // the only place the reason appears. + say(indent(refusal)) + } + + return out, nil +} + +// pushNode sends this node everything it should be. +// +// Given the long wait rather than the probe timeout: a push composes every declaration this node +// should hold, seals every secret in them and publishes them, and on a first node that is the +// slowest thing the mesh does. +func pushNode(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) { + said, err := control.within(o.Wait).tell(ctx, "push", o.Node) + if err != nil { + return "", fmt.Errorf( + "%w\n\nWhat was registered and assigned is registered and assigned, and this node has "+ + "not been sent it. Nothing is half-applied — a push the mesh refused sent nothing "+ + "at all. Fix what it named and run this installer again: it will find the module "+ + "already registered and try the push again", err) + } + say(" pushed " + o.Node) + return strings.TrimSpace(said), nil +} diff --git a/internal/bootstrap/publish.go b/internal/bootstrap/publish.go new file mode 100644 index 0000000..e7288b1 --- /dev/null +++ b/internal/bootstrap/publish.go @@ -0,0 +1,173 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" +) + +// ControlPlaneRepository is what the control plane's image is called in the mesh's own registry. +const ControlPlaneRepository = "mesh-control" + +// genesisTag is the tag the first push uses. +// +// A tag is not a pin and is never what anything is deployed from — the digest the registry assigns +// is (novox/hq ADR 0006). This exists so a person reading `/v2/mesh-control/tags/list` can see +// which image this mesh started from, and so the push has something to name. Everything downstream +// uses the digest that comes back. +const genesisTag = "genesis" + +// Published is what step 8 did. +type Published struct { + // Reference is `/mesh-control@sha256:…` — the first manifest digest this image has + // ever had, and the thing that makes the control plane an ordinary module. + Reference string + // Tagged is where it was pushed, tag and all. + Tagged string + // Already is true when the registry was already serving it and nothing was pushed. + Already bool +} + +// PublishControlPlane puts the carried image into the mesh's own registry and reads back its digest. +// +// **This is the pivot's hinge.** Every image must be pinned by digest, and a digest a pin can mean +// is one a REGISTRY assigned when something was pushed to it. The control plane's image is built +// from source and pushed nowhere, so it has none — which is why the substrate names it by the +// digest of its own configuration, and why that is legal exactly where nothing could have served +// one. The moment this push completes, that stops being true: the image has a manifest digest, so +// the control plane can be named the way every other module is named, so the mesh can build and +// roll out its own upgrades. If this step is skipped the machine still works and the mesh cannot +// upgrade itself, which is the check novox/hq ADR 0067 states: after installing, the running +// control plane must be pinned by a digest the mesh's own registry assigned, not by an image id. +// +// **It mirrors mesh-control's `internal/builder`.PublishImage rather than importing it.** Tag, +// push, read back `RepoDigests`, refuse anything without `@sha256:` — the same four steps, because +// there is exactly one right way to learn what a registry will serve something as, and it is to +// ask the registry. It is not imported because that code is tier 2: the host and its installer +// depend on nothing that must be installed first (novox/hq ADR 0041), and taking a dependency on +// the control plane's repository to raise the control plane would be the cycle this whole ADR is +// about, one layer up. The duplication is four commands, and it is deliberate. +// +// One difference, and it is a correction rather than a divergence: the digest is chosen from +// `RepoDigests` by repository instead of taken as element zero. An image that has been pushed to +// more than one registry has more than one entry, and element zero is then whichever the runtime +// happened to list first — which would pin this mesh to somebody else's registry, silently. +func PublishControlPlane(ctx context.Context, o Options, d Deps, imageID string, + say func(string)) (Published, error) { + + remote := o.Registry + "/" + ControlPlaneRepository + out := Published{Tagged: remote + ":" + genesisTag} + + // Asked first. A digest already served is a fact about the registry, and re-pushing an image + // the registry already holds is asking it to store what it already has under the name it + // already has. + if held, err := digestOf(ctx, o, d, remote); err != nil { + return out, err + } else if held != "" { + out.Reference, out.Already = held, true + say(" already published " + held) + return out, nil + } + + if _, err := d.Run(ctx, "docker", "tag", imageID, out.Tagged); err != nil { + return out, fmt.Errorf("cannot tag the carried image as %s: %w", out.Tagged, err) + } + if _, err := d.Run(ctx, "docker", "push", out.Tagged); err != nil { + return out, fmt.Errorf( + "the container runtime would not push %s: %w\n"+ + "The registry is plain HTTP and wants no credentials, deliberately — it is reached "+ + "over the mesh's own network, which is already the encrypted and authenticated "+ + "thing. A runtime refusing it for being insecure is refusing a registry on %s, "+ + "which it does not do for a loopback address", + out.Tagged, err, o.Registry) + } + + // Read back, from the registry's own answer rather than computed here. What matters is what + // the registry will serve for that reference, and only it can say (novox/hq ADR 0018). + pinned, err := digestOf(ctx, o, d, remote) + if err != nil { + return out, err + } + if pinned == "" { + return out, fmt.Errorf( + "%s was pushed and the registry does not serve it.\n"+ + "The next step names the control plane's module by the digest this was supposed to "+ + "produce, so there is nothing to name. Check `docker push` and "+ + "http://%s/v2/%s/tags/list", out.Tagged, o.Registry, ControlPlaneRepository) + } + out.Reference = pinned + say(" published " + pinned) + return out, nil +} + +// digestOf is what the registry serves this repository as, or empty if it serves it at all. +// +// Both halves are asked, because either alone lies. The registry's tag list says something was +// pushed and not what its digest is; the runtime's `RepoDigests` says what a digest was and not +// whether the registry still has it — a registry whose volume was recreated would leave the +// runtime remembering a digest nothing serves, and the module registered against it would pin the +// mesh to an image that cannot be pulled. +func digestOf(ctx context.Context, o Options, d Deps, remote string) (string, error) { + asking, cancel := context.WithTimeout(ctx, o.Timeout) + status, body, err := d.Fetch(asking, + "http://"+o.Registry+"/v2/"+ControlPlaneRepository+"/tags/list") + cancel() + if err != nil { + return "", fmt.Errorf("cannot ask the registry at %s what it holds: %w", o.Registry, err) + } + if status == http.StatusNotFound { + // Nothing has ever been pushed under this name. An answer, not a failure. + return "", nil + } + if status != http.StatusOK { + return "", fmt.Errorf("the registry answered %d when asked what it holds for %s", + status, ControlPlaneRepository) + } + var listed struct { + Tags []string `json:"tags"` + } + if err := json.Unmarshal([]byte(body), &listed); err != nil { + return "", fmt.Errorf("the registry's answer about %s is not readable: %w", + ControlPlaneRepository, err) + } + if !contains(listed.Tags, genesisTag) { + return "", nil + } + + // The registry has it. What digest, according to the runtime that pushed it. + reading, cancel := context.WithTimeout(ctx, o.Timeout) + out, err := d.Run(reading, "docker", "inspect", "--format", "{{json .RepoDigests}}", + remote+":"+genesisTag) + cancel() + if err != nil { + // The registry holds the tag and this machine's runtime does not hold the image. That + // happens on a re-run after the image was pruned, and it is not something to work around + // by trusting the tag: a tag can be made to point elsewhere. + return "", nil + } + var digests []string + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &digests); err != nil { + return "", fmt.Errorf("the runtime's answer about %s is not readable: %w", remote, err) + } + for _, digest := range digests { + if !strings.HasPrefix(digest, remote+"@sha256:") { + // Somebody else's registry serving the same image. Skipped rather than used: pinning + // this mesh's control plane to a registry it does not run is exactly the dependency + // the pivot exists to remove. + continue + } + return digest, nil + } + return "", nil +} + +func contains(values []string, want string) bool { + for _, v := range values { + if v == want { + return true + } + } + return false +} diff --git a/internal/bootstrap/publish_test.go b/internal/bootstrap/publish_test.go new file mode 100644 index 0000000..759414d --- /dev/null +++ b/internal/bootstrap/publish_test.go @@ -0,0 +1,197 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "testing" + "time" +) + +// Step 8 is the pivot's hinge (novox/hq ADR 0067): the carried image gets a manifest digest, which +// is the first one it has ever had, and that is what lets the control plane be named the way every +// other module is named. These tests defend how that digest is learned, because a wrong one pins +// the mesh to an image nothing on this machine serves. + +func publishing(t *testing.T, fetch func(string) (int, string, error), + run func(name string, args []string) (string, error)) (Options, Deps, *asked) { + t.Helper() + runtime := &asked{answer: run} + return Options{ + Registry: "127.0.0.1:5000", + Timeout: time.Second, + Wait: 0, + }, Deps{ + Run: runtime.run, + Fetch: func(_ context.Context, url string) (int, string, error) { + return fetch(url) + }, + }, runtime +} + +// Nothing has ever been pushed under this name, so the registry says 404 — and that is an answer, +// not a failure. An installer that treated it as one would refuse on the first run of the step it +// exists to perform. +func TestAnImageNoRegistryHasEverHeldIsPushed(t *testing.T) { + pushed := false + o, d, runtime := publishing(t, + func(string) (int, string, error) { + if !pushed { + return http.StatusNotFound, "", nil + } + return http.StatusOK, `{"name":"mesh-control","tags":["genesis"]}`, nil + }, + func(_ string, args []string) (string, error) { + switch args[0] { + case "tag": + return "", nil + case "push": + pushed = true + return "", nil + case "inspect": + return `["127.0.0.1:5000/mesh-control@sha256:` + strings.Repeat("a", 64) + `"]`, nil + } + return "", fmt.Errorf("unexpected: %v", args) + }) + + out, err := PublishControlPlane(context.Background(), o, d, held, func(string) {}) + if err != nil { + t.Fatal(err) + } + if out.Already { + t.Error("an image no registry held was reported as already published") + } + if !strings.HasPrefix(out.Reference, "127.0.0.1:5000/mesh-control@sha256:") { + t.Errorf("the control plane is pinned as %q", out.Reference) + } + if !runtime.ran("docker push 127.0.0.1:5000/mesh-control:genesis") { + t.Errorf("nothing was pushed: %v", runtime.commands) + } +} + +// An image the registry already serves is not pushed again, and says so. Blobs are named by their +// content, so re-pushing is asking a registry to store what it already has under the name it +// already has — and the installer is run over and over. +func TestAnImageTheRegistryAlreadyServesIsNotPushedAgain(t *testing.T) { + o, d, runtime := publishing(t, + func(string) (int, string, error) { + return http.StatusOK, `{"name":"mesh-control","tags":["genesis"]}`, nil + }, + func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return `["127.0.0.1:5000/mesh-control@sha256:` + strings.Repeat("b", 64) + `"]`, nil + } + return "", fmt.Errorf("unexpected: %v", args) + }) + + out, err := PublishControlPlane(context.Background(), o, d, held, func(string) {}) + if err != nil { + t.Fatal(err) + } + if !out.Already { + t.Error("an image the registry already serves was not reported as already published") + } + if runtime.ran("docker push") { + t.Errorf("it was pushed again: %v", runtime.commands) + } +} + +// **The digest is chosen by repository, not taken as element zero.** An image that has been pushed +// to more than one registry has more than one entry, and element zero is whichever the runtime +// listed first — which would pin this mesh's control plane to somebody else's registry, silently, +// which is the dependency the whole pivot exists to remove. +func TestTheDigestComesFromThisMeshsOwnRegistry(t *testing.T) { + elsewhere := "some.other.registry/mesh-control@sha256:" + strings.Repeat("c", 64) + ours := "127.0.0.1:5000/mesh-control@sha256:" + strings.Repeat("d", 64) + + o, d, _ := publishing(t, + func(string) (int, string, error) { + return http.StatusOK, `{"tags":["genesis"]}`, nil + }, + func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return `["` + elsewhere + `","` + ours + `"]`, nil + } + return "", fmt.Errorf("unexpected: %v", args) + }) + + out, err := PublishControlPlane(context.Background(), o, d, held, func(string) {}) + if err != nil { + t.Fatal(err) + } + if out.Reference != ours { + t.Errorf("the control plane is pinned as %q, and this mesh's registry serves %q", + out.Reference, ours) + } +} + +// A push that produced no digest this mesh's registry serves is refused, and the refusal says what +// depends on it. The next step names the control plane's module by that digest, so there would be +// nothing to name — and finding that out one step later would mean registering a module pinned to +// an empty string. +func TestAPushThatProducedNoDigestIsRefused(t *testing.T) { + pushed := false + o, d, _ := publishing(t, + func(string) (int, string, error) { + if !pushed { + return http.StatusNotFound, "", nil + } + // Pushed, and the registry still does not list it. + return http.StatusOK, `{"tags":[]}`, nil + }, + func(_ string, args []string) (string, error) { + if args[0] == "push" { + pushed = true + } + return "", nil + }) + + _, err := PublishControlPlane(context.Background(), o, d, held, func(string) {}) + if err == nil { + t.Fatal("a push that produced no digest was accepted") + } + if !strings.Contains(err.Error(), "does not serve it") { + t.Errorf("the refusal does not say what is missing: %v", err) + } +} + +// A tag is not a pin. If the runtime answers with something that is not pinned by digest, it is +// not used — a tag can be made to point at a different image, and this reference is applied on +// machines with no mesh to ask about anything (novox/hq ADR 0006). +func TestATagIsNotAPin(t *testing.T) { + o, d, _ := publishing(t, + func(string) (int, string, error) { + return http.StatusOK, `{"tags":["genesis"]}`, nil + }, + func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return `["127.0.0.1:5000/mesh-control:genesis"]`, nil + } + return "", nil + }) + + out, err := PublishControlPlane(context.Background(), o, d, held, func(string) {}) + if err == nil { + t.Fatalf("a tag was accepted as a pin: %q", out.Reference) + } +} + +// A registry that cannot be reached at all is said so plainly rather than becoming a push that +// fails for a reason nobody can read. +func TestARegistryThatCannotBeAskedIsSaidSo(t *testing.T) { + o, d, _ := publishing(t, + func(string) (int, string, error) { + return 0, "", errors.New("connection refused") + }, + func(string, []string) (string, error) { return "", nil }) + + _, err := PublishControlPlane(context.Background(), o, d, held, func(string) {}) + if err == nil { + t.Fatal("a registry that refused the connection was treated as empty") + } + if !strings.Contains(err.Error(), "cannot ask the registry") { + t.Errorf("the refusal does not say the registry could not be asked: %v", err) + } +} diff --git a/internal/bootstrap/registry.go b/internal/bootstrap/registry.go new file mode 100644 index 0000000..637d9e7 --- /dev/null +++ b/internal/bootstrap/registry.go @@ -0,0 +1,204 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" +) + +// RegistryModule is the module that provides the mesh's artifact store. +const RegistryModule = "registry" + +// registryResource is the id of the resource in that module's manifest that runs the registry. +// What the container is CALLED is read from the manifest rather than assumed, because the name is +// the catalogue's to choose and the installer only has to know which resource to wait for. +const registryResource = "store" + +// Registry is what step 7 did. +type Registry struct { + Installed + // Container is the container the manifest declares, confirmed running. + Container string + // Address is host:port the registry answers on, as this machine reaches it. + Address string + // Answered is the status the registry's own `/v2/` gave back. + Answered int +} + +// InstallRegistry gives this mesh somewhere to put images. +// +// **Its image is upstream and it is never built.** novox/hq 04-ISSUES/029 is the whole reason this +// step exists in this position: a module that provides the artifact store cannot be delivered +// through the artifact store, so the registry is the one module whose image is pulled from the +// internet like the store and the broker before it. A manifest carrying the catalogue's +// placeholder digest here would mean somebody had made it buildable, which is the cycle again — +// so it is refused rather than pulled. +// +// **No credentials, and that is deliberate.** The registry is reached over the mesh's own private +// network, which is already the encrypted and authenticated thing; a second layer inside it would +// be certificates to issue and rotate for no property the first does not have (mesh-control's +// `internal/builder`, which pushes to it the same way). So there is nothing here to configure and +// nothing to seal — which is also why step 8 can push without the mesh having issued anything. +// +// **It is verified by asking it, not by looking at it.** A container that is up is not a registry +// that serves: `/v2/` is the registry API's own "yes, I am one and I am ready", and it is the +// question step 8 depends on the answer to. +func InstallRegistry(ctx context.Context, o Options, d Deps, control controlPlane, + say func(string)) (Registry, error) { + + out := Registry{Address: o.Registry} + + manifest, err := readManifest(o.Catalogue, RegistryModule) + if err != nil { + return out, err + } + container, image, err := containerIn(manifest, registryResource) + if err != nil { + return out, err + } + if strings.Contains(image, placeholderDigest) { + return out, fmt.Errorf( + "the %s module's image is %q, which is the catalogue's placeholder for something the "+ + "mesh builds and pushes.\n"+ + "This module is the one that cannot work that way: it PROVIDES the place built "+ + "images go, so it can never be delivered through it (novox/hq 04-ISSUES/029). Its "+ + "image is upstream and pinned in the manifest", RegistryModule, image) + } + out.Container = container + say(" registry image " + image + " — upstream, never built") + + installed, err := installModule(ctx, o, control, RegistryModule, manifest, say) + out.Installed = installed + if err != nil { + return out, err + } + + // Read back, in two stages, because they fail differently. A container that never appears is + // a declaration that did not reach this node or an image that would not pull; a container + // that is up and does not answer is a registry that started and failed. + if err := waitForContainer(ctx, control.run, o.Timeout, o.Wait, container, say); err != nil { + return out, err + } + status, err := waitForTheRegistry(ctx, d, o, say) + if err != nil { + return out, err + } + out.Answered = status + return out, nil +} + +// waitForTheRegistry asks `/v2/` until it answers. +func waitForTheRegistry(ctx context.Context, d Deps, o Options, say func(string)) (int, error) { + where := "http://" + o.Registry + "/v2/" + + deadline := time.Now().Add(o.Wait) + var last string + for { + asking, cancel := context.WithTimeout(ctx, o.Timeout) + status, _, err := d.Fetch(asking, where) + cancel() + switch { + case err != nil: + last = err.Error() + case status == http.StatusOK: + say(fmt.Sprintf(" replies %s answered %d", where, status)) + return status, nil + default: + // A registry that answers 401 is one that wants credentials, which this one is + // configured not to. Reported as what it said rather than retried into a timeout. + last = fmt.Sprintf("it answered %d", status) + } + + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + return 0, ctx.Err() + case <-time.After(answerEvery): + } + } + return 0, fmt.Errorf( + "the registry's container is running and %s does not answer, after waiting %s: %s\n"+ + "Running is not serving. `/v2/` is the registry API saying it is ready, and the next "+ + "step pushes the control plane's image to it — so this is refused here rather than "+ + "discovered inside a `docker push`. `docker logs mesh-registry` says what it did", + where, o.Wait, last) +} + +// waitForContainer waits for a container the mesh was asked to create to be running. +// +// Unlike the substrate's own verify, this one waits: the mesh applies through a node's host, over +// the broker, asynchronously. A push that the control plane accepted has not yet happened on the +// machine, and refusing on the first look would refuse every correct install. +func waitForContainer(ctx context.Context, run Runner, probe, wait time.Duration, name string, + say func(string)) error { + + deadline := time.Now().Add(wait) + var last string + for { + state, err := containerRunning(ctx, run, probe, name) + switch { + case err != nil: + last = "it is not there at all" + case state.running: + say(" running " + name) + return nil + default: + last = "it is " + state.status + } + + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(answerEvery): + } + } + return fmt.Errorf( + "the mesh accepted the push and %q is not running after %s: %s\n"+ + "The control plane sends a declaration over the broker and this node's host applies "+ + "it, so the two ends fail differently: `mesh-host` on this machine says what it made "+ + "of the declaration, and `status` on the control plane says whether it was collected "+ + "at all", name, wait, last) +} + +// containerIn finds one container resource in a module manifest and gives back its name and image. +// +// It reads the manifest as data rather than through the catalogue's own parser, because that +// parser lives in the control plane and the host depends on nothing installed first +// (novox/hq ADR 0041) — importing it would put tier 2 inside tier 0. What is read here is two +// fields of a shape the catalogue owns; the manifest is handed to the control plane unchanged, and +// it is the control plane's `module add` that judges whether it is a manifest at all. +func containerIn(manifest []byte, id string) (name, image string, err error) { + var m struct { + Module string `json:"module"` + Resources []struct { + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + Image string `json:"image"` + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + return "", "", fmt.Errorf("this manifest is not readable as JSON: %w", err) + } + var containers []string + for _, r := range m.Resources { + if r.Type != "container" { + continue + } + containers = append(containers, r.ID) + if r.ID == id { + return r.Name, r.Image, nil + } + } + return "", "", fmt.Errorf( + "the %s module declares no container %q, so the installer does not know what to wait for. "+ + "It declares: %s", m.Module, id, strings.Join(containers, ", ")) +} diff --git a/internal/bootstrap/registry_test.go b/internal/bootstrap/registry_test.go new file mode 100644 index 0000000..52f923d --- /dev/null +++ b/internal/bootstrap/registry_test.go @@ -0,0 +1,216 @@ +package bootstrap + +import ( + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// Step 7 installs the one module whose image can never come from the mesh's own registry, because +// it IS the mesh's own registry (novox/hq 04-ISSUES/029). These tests defend that, and defend the +// distinction the whole verify layer of this program is built on: a container that is up is not a +// service that answers. + +// catalogueWith writes a fake catalogue checkout holding one module's manifest. +// +// A fixture here rather than the real catalogue, unlike the substrate example the rewrite tests +// use: the catalogue is a different repository on a different branch, and a test that read it +// would pass or fail according to what somebody else had checked out. +func catalogueWith(t *testing.T, module, manifest string) string { + t.Helper() + root := t.TempDir() + dir := filepath.Join(root, catalogueDir, module) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + return root +} + +const upstreamRegistryManifest = `{ + "module": "registry", + "version": "1", + "provides": [{"name": "artifact-store", "scope": "mesh"}], + "capabilities": ["container-runtime"], + "resources": [ + {"id": "state", "type": "directory", "path": "/var/lib/mesh/registry", "mode": "0700"}, + {"id": "store", "type": "container", "name": "mesh-registry", + "image": "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373", + "ports": ["5000:5000"]} + ] +}` + +// aMeshThatAgrees answers every command the installer issues at steps 7 and 9 the way a working +// mesh would, except for whatever a test overrides. +func aMeshThatAgrees(answers map[string]string) func(string, []string) (string, error) { + return func(name string, args []string) (string, error) { + joined := strings.Join(args, " ") + for fragment, said := range answers { + if strings.Contains(joined, fragment) { + return said, nil + } + } + switch { + case name != "docker": + return "", fmt.Errorf("unexpected program %q", name) + case args[0] == "cp": + return "", nil + case args[0] == "inspect": + return "true running\n", nil + case args[0] == "exec": + return "", nil + } + return "", fmt.Errorf("unexpected: %v", args) + } +} + +func installing(t *testing.T, catalogue string) Options { + t.Helper() + return Options{ + Node: "anchor", + Catalogue: catalogue, + Registry: "127.0.0.1:5000", + Timeout: time.Second, + Wait: 0, + } +} + +// A container that is up is not a registry that serves. `/v2/` is the registry API's own "yes, I +// am one and I am ready", and the step after this pushes to it — so it is refused here rather than +// discovered inside a `docker push`. +func TestARegistryContainerThatIsUpIsNotARegistryThatServes(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + runtime := &asked{answer: aMeshThatAgrees(nil)} + deps := Deps{ + Run: runtime.run, + Fetch: func(context.Context, string) (int, string, error) { + return http.StatusInternalServerError, "", nil + }, + } + + _, err := InstallRegistry(context.Background(), + installing(t, catalogueWith(t, RegistryModule, upstreamRegistryManifest)), + deps, controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err == nil { + t.Fatal("a registry whose container is up and which answers 500 was accepted") + } + for _, wanted := range []string{"/v2/", "Running is not serving"} { + if !strings.Contains(err.Error(), wanted) { + t.Errorf("the refusal does not mention %q:\n%v", wanted, err) + } + } +} + +// The whole of step 7, against a mesh that agrees: registered, assigned, pushed, up, and answering. +func TestARegistryThatAnswersIsAccepted(t *testing.T) { + runtime := &asked{answer: aMeshThatAgrees(nil)} + deps := Deps{ + Run: runtime.run, + Fetch: func(context.Context, string) (int, string, error) { + return http.StatusOK, "{}", nil + }, + } + + out, err := InstallRegistry(context.Background(), + installing(t, catalogueWith(t, RegistryModule, upstreamRegistryManifest)), + deps, controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err != nil { + t.Fatal(err) + } + if out.Container != "mesh-registry" { + t.Errorf("the registry's container is %q", out.Container) + } + if out.Answered != http.StatusOK { + t.Errorf("the registry answered %d", out.Answered) + } + // Registered, assigned and pushed, through the same three commands a person types. + for _, wanted := range []string{ + "module add /registry-module.json", + "assign anchor registry", + "push anchor", + } { + if !runtime.ran(wanted) { + t.Errorf("the installer never ran %q: %v", wanted, runtime.commands) + } + } +} + +// **The registry's image is upstream and it is never built.** A manifest carrying the catalogue's +// placeholder digest would mean somebody had made this module buildable — which is the cycle +// novox/hq 04-ISSUES/029 settled: a module that provides the artifact store cannot be delivered +// through the artifact store. +func TestARegistryManifestThatWantsBuildingIsRefused(t *testing.T) { + wants := strings.Replace(upstreamRegistryManifest, + "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373", + "mesh-runtime-registry@"+placeholderDigest, 1) + + runtime := &asked{answer: aMeshThatAgrees(nil)} + _, err := InstallRegistry(context.Background(), + installing(t, catalogueWith(t, RegistryModule, wants)), + Deps{Run: runtime.run}, controlPlane{container: "temp-mesh-control", run: runtime.run}, + func(string) {}) + if err == nil { + t.Fatal("a registry manifest naming an image the mesh would have to build was accepted") + } + if !strings.Contains(err.Error(), "04-ISSUES/029") { + t.Errorf("the refusal does not name the decision it rests on: %v", err) + } + if runtime.ran("module add") { + t.Error("it was registered anyway") + } +} + +// A catalogue that is not there is said plainly, with what --catalog is. This is the most likely +// mistake anybody makes at this step and the least interesting to debug. +func TestACatalogueThatIsNotThereIsSaidPlainly(t *testing.T) { + runtime := &asked{answer: aMeshThatAgrees(nil)} + _, err := InstallRegistry(context.Background(), + installing(t, filepath.Join(t.TempDir(), "nowhere")), + Deps{Run: runtime.run}, controlPlane{container: "temp-mesh-control", run: runtime.run}, + func(string) {}) + if err == nil { + t.Fatal("a catalogue that does not exist was accepted") + } + if !strings.Contains(err.Error(), "--catalog") { + t.Errorf("the refusal does not say what to fix: %v", err) + } +} + +// A refusal from the control plane is repeated verbatim. mesh-control refuses in paragraphs — +// "nothing provides route, wanted by registry" — and an installer that reported "exit status 1" +// would throw away the only thing a person can act on. +func TestWhatTheMeshRefusedIsRepeated(t *testing.T) { + refusal := "nothing provides \"route\", wanted by registry" + runtime := &asked{answer: func(name string, args []string) (string, error) { + if strings.Contains(strings.Join(args, " "), "push") { + return refusal, fmt.Errorf("exit status 1") + } + return aMeshThatAgrees(nil)(name, args) + }} + + _, err := InstallRegistry(context.Background(), + installing(t, catalogueWith(t, RegistryModule, upstreamRegistryManifest)), + Deps{Run: runtime.run}, controlPlane{container: "temp-mesh-control", run: runtime.run, + timeout: time.Second}, func(string) {}) + if err == nil { + t.Fatal("a push the mesh refused was reported as successful") + } + if !strings.Contains(err.Error(), refusal) { + t.Errorf("what the mesh said is not in the failure:\n%v", err) + } + if !strings.Contains(err.Error(), "run this installer again") { + t.Errorf("the failure does not say a re-run continues from here:\n%v", err) + } +} diff --git a/internal/bootstrap/retire.go b/internal/bootstrap/retire.go new file mode 100644 index 0000000..a84055d --- /dev/null +++ b/internal/bootstrap/retire.go @@ -0,0 +1,291 @@ +package bootstrap + +import ( + "bytes" + "context" + "fmt" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/system" +) + +// Retired is what step 10 did. +type Retired struct { + // Container is the temporary control plane that was dropped. + Container string + // Gone is true when the machine no longer has it. + Gone bool + // Already is true when it was gone before this step ran. + Already bool + // Bundle is where the bundle without it was written. + Bundle string + // Removed is how many resources the re-apply reported removing. + Removed int +} + +// RetireTheTemporaryControlPlane drops it from the bundle and lets the host take it away. +// +// **Destruction by omission, which is the host's ordinary behaviour and not a new mechanism.** The +// host owns what it has applied and removes what it owns and is no longer declared. So retiring the +// temporary control plane is not a verb anybody had to invent: the bundle stops declaring it, the +// bundle is applied again, and the removal pass does what it does for every other resource that +// leaves a declaration. +// +// That is the whole of why the rename at step 3 mattered. Had the substrate and the module both +// called their container `mesh-control`, this apply would have removed the module's container — +// the host would have been asked to take away something it believed it owned, and it would have +// been right. Two names, two owners, and the removal is unambiguous. +// +// **It is the last step for a reason.** Until step 9 has a control plane that answers, the +// temporary one is the only thing that can tell this machine anything, and a machine left with no +// control plane cannot be fixed remotely (novox/hq ADR 0067). So this runs after the permanent one +// has been proved, and a run interrupted before it leaves two control planes, which is untidy and +// harmless — a re-run reaches this step and finishes. +func RetireTheTemporaryControlPlane(ctx context.Context, o Options, sys system.System, + produced []byte, run Runner, say func(string)) (Retired, error) { + + out := Retired{Bundle: o.Out} + + current, err := declaration.ParseFileTrusted(produced) + if err != nil { + return out, fmt.Errorf("the bundle this installer produced is not a declaration: %w", err) + } + temporary, err := controlPlaneIn(current) + if err != nil { + // The bundle already declares no control plane, which is what a re-run after this step + // finds. Nothing to drop, and nothing to be alarmed about. + say(" already dropped the bundle declares no temporary control plane") + out.Already = true + return out, nil + } + out.Container = temporary.Name + + // Textual, for the reason the rewrite at step 3 is textual: the produced bundle is meant to be + // READ, and a person coming to a machine after a pivot should be able to open the file the + // installer applied and see the substrate they recognise with the control plane gone from it. + // Re-serialising a parsed declaration would drop every comment in it. + bundle, err := removeResource(produced, ControlPlaneID) + if err != nil { + return out, err + } + without, err := declaration.ParseFileTrusted(bundle) + if err != nil { + return out, fmt.Errorf( + "taking the temporary control plane out of the bundle broke it: %w", err) + } + if _, err := controlPlaneIn(without); err == nil { + return out, fmt.Errorf( + "the bundle still declares %q after it was taken out, so nothing was removed and the "+ + "apply below would change nothing", ControlPlaneID) + } + if err := writeBundleFile(o.Out, bundle); err != nil { + return out, err + } + say(fmt.Sprintf(" wrote %s (%d resources) — without %s", + o.Out, len(without.Resources), temporary.Name)) + + // Applied the same way everything else here is applied, under the same origin, against the + // same state file. What makes this a removal rather than a no-op is that the state file + // records the container as something this installer applied, and the declaration no longer + // asks for it. + report, err := ApplyBundle(ctx, o, sys, without, run, say) + if err != nil { + return out, fmt.Errorf( + "%w\n\nThe permanent control plane is running and the temporary one is still here. "+ + "That is untidy and it is not broken: two control planes on one mesh are both "+ + "stateless and both correct. Run this installer again to finish", err) + } + for _, outcome := range report.Outcomes { + if outcome.Action == "removed" { + out.Removed++ + } + } + + // Read back. A removal that reported success and left the container running would leave two + // control planes consuming the same broker queues for ever, which is the state this step + // exists to end. + gone, err := isGone(ctx, run, o.Timeout, o.Wait, temporary.Name) + if err != nil { + return out, err + } + out.Gone = gone + if !gone { + return out, fmt.Errorf( + "the apply reported the temporary control plane removed and %q is still running.\n"+ + "Two control planes are consuming this mesh's broker queues. Neither is wrong and "+ + "the mesh is not damaged, but the pivot is not finished: `docker rm -f %s` ends "+ + "it, and this installer will then agree", temporary.Name, temporary.Name) + } + say(" gone " + temporary.Name) + return out, nil +} + +// removeResource takes one resource out of a bundle's text, comments and all. +// +// It walks the `resources` array counting braces, skipping over strings and comments so that a +// `//` inside a connection string is not read as the start of one — the substrate's own bundle +// contains `postgres://…` several times, and a scanner that did not know the difference would +// treat the rest of the line as a comment and lose a brace. +// +// What is removed is the element AND whatever precedes it back to the previous element, which is +// where the comment explaining it lives. A comment that outlives the thing it describes is worse +// than no comment: it is the file telling somebody the machine has a control plane it does not. +func removeResource(bundle []byte, id string) ([]byte, error) { + array := indexOutsideStrings(bundle, `"resources"`) + if array < 0 { + return nil, fmt.Errorf("this bundle has no resources array, so there is nothing to take out of it") + } + open := indexOutsideStrings(bundle[array:], "[") + if open < 0 { + return nil, fmt.Errorf("this bundle's resources are not a list") + } + open += array + + depth, from := 0, -1 + previous := open + inString, escaped, inLine, inBlock := false, false, false, false + for i := open + 1; i < len(bundle); i++ { + c := bundle[i] + switch { + case escaped: + escaped = false + case inString && c == '\\': + escaped = true + case inString: + if c == '"' { + inString = false + } + case inLine: + if c == '\n' { + inLine = false + } + case inBlock: + if c == '*' && i+1 < len(bundle) && bundle[i+1] == '/' { + inBlock, i = false, i+1 + } + case c == '"': + inString = true + case c == '/' && i+1 < len(bundle) && bundle[i+1] == '/': + inLine, i = true, i+1 + case c == '/' && i+1 < len(bundle) && bundle[i+1] == '*': + inBlock, i = true, i+1 + case c == '{': + if depth == 0 { + from = i + } + depth++ + case c == '}': + depth-- + if depth != 0 { + break + } + if isResource(bundle[from:i+1], id) { + return cut(bundle, previous, from, i+1), nil + } + previous = i + 1 + from = -1 + case c == ']' && depth == 0: + return nil, fmt.Errorf( + "this bundle declares no %q, so there is nothing to take out of it", id) + } + } + return nil, fmt.Errorf("this bundle's resources list does not end") +} + +// isResource reports whether one resource's text is the one wanted. +// +// Whitespace-insensitive on the pair, quotes included, so `"id": "control-plane"` and +// `"id":"control-plane"` are the same answer and `"id": "control-planes"` is not. +func isResource(resource []byte, id string) bool { + var tight []byte + for _, c := range resource { + if c != ' ' && c != '\t' && c != '\n' && c != '\r' { + tight = append(tight, c) + } + } + return bytes.Contains(tight, []byte(`"id":"`+id+`"`)) +} + +// cut removes an element and what leads up to it, leaving the list valid. +// +// Whether the comma before or the comma after goes depends on where the element sits: an element +// with something before it takes the comma that joined them, and the first element takes the one +// after it. Getting this wrong produces a trailing comma, which is JSON nothing will parse — +// caught by the re-parse either way, and better not produced. +func cut(bundle []byte, previous, from, to int) []byte { + start := from + for i := previous; i < from; i++ { + if bundle[i] == ',' { + start = i + break + } + } + end := to + if start == from { + // Nothing before it, so the comma that follows is the one that would be left dangling. + for i := to; i < len(bundle); i++ { + if bundle[i] == ',' { + end = i + 1 + break + } + if bundle[i] == ']' { + break + } + } + } + out := make([]byte, 0, len(bundle)) + out = append(out, bundle[:start]...) + return append(out, bundle[end:]...) +} + +// indexOutsideStrings finds a fragment that is not inside a JSON string. +func indexOutsideStrings(haystack []byte, needle string) int { + inString, escaped := false, false + for i := 0; i < len(haystack); i++ { + switch { + case escaped: + escaped = false + continue + case haystack[i] == '\\' && inString: + escaped = true + continue + case haystack[i] == '"': + // The needle may itself start with a quote, so the match is tried before the quote is + // consumed. + if !inString && bytes.HasPrefix(haystack[i:], []byte(needle)) { + return i + } + inString = !inString + continue + case inString: + continue + } + if bytes.HasPrefix(haystack[i:], []byte(needle)) { + return i + } + } + return -1 +} + +// isGone waits for a container to stop existing. +// +// Waited for rather than asked once, because a container being removed is a container that is +// stopping first, and a runtime answers about it until it has finished. +func isGone(ctx context.Context, run Runner, probe, wait time.Duration, name string) (bool, error) { + deadline := time.Now().Add(wait) + for { + if _, err := containerRunning(ctx, run, probe, name); err != nil { + // The runtime does not know it. That is the answer being waited for. + return true, nil + } + if time.Now().After(deadline) { + return false, nil + } + select { + case <-ctx.Done(): + return false, ctx.Err() + case <-time.After(answerEvery): + } + } +} diff --git a/internal/bootstrap/retire_test.go b/internal/bootstrap/retire_test.go new file mode 100644 index 0000000..efdec44 --- /dev/null +++ b/internal/bootstrap/retire_test.go @@ -0,0 +1,165 @@ +package bootstrap + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +// Retirement is destruction by omission, which is the host's ordinary behaviour: it owns what it +// applied and removes what it owns and is no longer declared. These tests defend the bundle +// surgery that expresses it, because a bundle that came out of it unparseable would be found by +// the apply — after the file on the machine had already been replaced. + +func produced(t *testing.T) []byte { + t.Helper() + out, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + return out.Bundle +} + +// The temporary control plane leaves the bundle, everything else stays, and what is left parses. +func TestTheTemporaryControlPlaneLeavesTheBundleAndNothingElseDoes(t *testing.T) { + before, err := declaration.ParseFileTrusted(produced(t)) + if err != nil { + t.Fatal(err) + } + shorter, err := removeResource(produced(t), ControlPlaneID) + if err != nil { + t.Fatal(err) + } + after, err := declaration.ParseFileTrusted(shorter) + if err != nil { + t.Fatalf("the bundle without the control plane does not parse: %v\n%s", err, shorter) + } + if len(after.Resources) != len(before.Resources)-1 { + t.Fatalf("the bundle went from %d resources to %d, and one was removed", + len(before.Resources), len(after.Resources)) + } + if _, err := controlPlaneIn(after); err == nil { + t.Error("the bundle still declares a control plane") + } + // The store and the broker are still exactly what they were. A retirement that took the + // substrate with it would leave the machine with a module and nothing under it. + for id, name := range containerNames(before) { + if id == ControlPlaneID { + continue + } + if containerNames(after)[id] != name { + t.Errorf("%s was lost or renamed by the retirement", id) + } + } +} + +// **A `//` inside a string is not a comment.** The substrate's own bundle carries +// `postgres://…` several times, and a scanner that read the rest of those lines as a comment +// would lose braces and cut the wrong thing out — silently, because what it produced would still +// look like a file. +func TestASchemeInsideAStringIsNotReadAsAComment(t *testing.T) { + shorter, err := removeResource(produced(t), ControlPlaneID) + if err != nil { + t.Fatal(err) + } + after, err := declaration.ParseFileTrusted(shorter) + if err != nil { + t.Fatal(err) + } + // The migration action, which is the resource holding the most `://` of anything here, is + // still whole. + found := false + for _, r := range after.Resources { + if r.Identity() == "context-schemas" { + found = true + } + } + if !found { + t.Error("the resource full of connection strings did not survive the removal") + } +} + +// Removing the FIRST element takes the comma after it rather than the comma before it, because +// there is no comma before it. Getting this wrong produces a leading comma, which is JSON nothing +// parses — and the file would already have been written. +func TestRemovingTheFirstResourceLeavesAValidList(t *testing.T) { + shorter, err := removeResource(produced(t), "container-runtime") + if err != nil { + t.Fatal(err) + } + after, err := declaration.ParseFileTrusted(shorter) + if err != nil { + t.Fatalf("removing the first resource broke the bundle: %v\n%s", err, shorter) + } + for _, r := range after.Resources { + if r.Identity() == "container-runtime" { + t.Error("the first resource is still there") + } + } +} + +// A bundle that declares no such resource is refused rather than silently returned unchanged. A +// removal that removed nothing and reported success would leave the apply below with nothing to +// do and the installer claiming a pivot it did not finish. +func TestRemovingSomethingThatIsNotThereIsRefused(t *testing.T) { + if _, err := removeResource(produced(t), "nothing-of-the-sort"); err == nil { + t.Fatal("a bundle was reported to have had a resource removed that it never declared") + } +} + +// A re-run after the retirement finds a bundle with no control plane in it and says so, rather +// than failing. This is the idempotence of the last step, and it is the one a person is most +// likely to exercise: the pivot ends here, so a re-run to check ends here too. +func TestRetiringABundleThatAlreadyHasNoControlPlaneIsAlreadyDone(t *testing.T) { + shorter, err := removeResource(produced(t), ControlPlaneID) + if err != nil { + t.Fatal(err) + } + var said []string + out, err := RetireTheTemporaryControlPlane(context.Background(), Options{}, + nil, shorter, nil, func(line string) { said = append(said, line) }) + if err != nil { + t.Fatal(err) + } + if !out.Already { + t.Error("a bundle with no control plane in it was not reported as already retired") + } + if !strings.Contains(strings.Join(said, "\n"), "already dropped") { + t.Errorf("the run does not say it was already done: %v", said) + } +} + +// A container the runtime still knows about after the apply is a pivot that did not finish. Two +// control planes on one mesh are both correct and neither is wrong — but the temporary one was +// supposed to go, and saying it went when it did not is the fault this project keeps naming. +func TestAContainerStillThereAfterRemovalIsNotGone(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + stillThere := &asked{answer: func(_ string, _ []string) (string, error) { + return "true running\n", nil + }} + gone, err := isGone(context.Background(), stillThere.run, time.Second, 0, "temp-mesh-control") + if err != nil { + t.Fatal(err) + } + if gone { + t.Error("a container the runtime still describes was reported gone") + } + + removed := &asked{answer: func(_ string, _ []string) (string, error) { + return "", errors.New("No such object: temp-mesh-control") + }} + gone, err = isGone(context.Background(), removed.run, time.Second, 0, "temp-mesh-control") + if err != nil { + t.Fatal(err) + } + if !gone { + t.Error("a container the runtime does not know about was not reported gone") + } +} diff --git a/internal/bootstrap/talk.go b/internal/bootstrap/talk.go new file mode 100644 index 0000000..eab5a14 --- /dev/null +++ b/internal/bootstrap/talk.go @@ -0,0 +1,119 @@ +package bootstrap + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// controlPlane is the running control plane, asked things. +// +// **Through `docker exec`, not over a network.** The control plane listens on nothing — `serve` is +// a broker consumer, and every administrative verb is a subcommand of the same binary that opens +// the stores directly (mesh-control's own usage). So the way to tell a mesh anything, from the +// machine the mesh is on, is to run its binary inside its own container. That is also what the lab +// does, and having the installer and the lab drive the mesh identically is the point: the lab is +// meant to exercise the installer, not a second procedure that resembles it. +// +// It carries which container, because the whole pivot turns on there being two of them: the +// substrate's `temp-mesh-control` for steps 6 to 9, and the module's `mesh-control` afterwards. +type controlPlane struct { + container string + run Runner + timeout time.Duration +} + +// within is the same control plane, asked with a different patience. +// +// A copy rather than a field somebody sets, so a slow command cannot leave every command after it +// slow: the caller that needs the long wait says so on the call. +func (c controlPlane) within(timeout time.Duration) controlPlane { + c.timeout = timeout + return c +} + +// tell runs a mesh-control subcommand and gives back what it said. +// +// The failure carries the command AND the output. A mesh-control refusal is a paragraph explaining +// what is wrong — "nothing provides route, wanted by registry" — and an installer that reported +// only "exit status 1" would throw away the one thing a person needs. +func (c controlPlane) tell(ctx context.Context, args ...string) (string, error) { + asking, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + + out, err := c.run(asking, "docker", append( + []string{"exec", c.container, controlPlaneBinary}, args...)...) + if err != nil { + return out, fmt.Errorf("`%s %s` was refused: %w\n%s", + c.container, strings.Join(args, " "), err, indent(strings.TrimSpace(out))) + } + return out, nil +} + +// carry puts a file inside the control plane's container. +// +// **Because every command that takes a file reads it from its own filesystem.** `module add +// ` and `secret accept --from ` open a path, and the process doing the opening is +// inside the container. The installer is not. So the file is copied in first, exactly as the lab +// does it. +// +// The image is `FROM scratch` and has no shell, so nothing inside can move a file, change its mode +// or clean up after itself. What is copied in stays until the container is replaced — which, for +// the temporary control plane, is a container that gets removed at step 10 and takes its contents +// with it. +func (c controlPlane) carry(ctx context.Context, local, remote string) error { + asking, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + + if _, err := c.run(asking, "docker", "cp", local, c.container+":"+remote); err != nil { + return fmt.Errorf("cannot put %s into %s at %s: %w", local, c.container, remote, err) + } + return nil +} + +// carrying writes bytes to a temporary file on the machine and copies them into the container. +// +// **0644, and that is not carelessness — 0600 would break it.** `docker cp` keeps the ownership and +// mode a file had outside, the control plane's image runs as 65534, and the process that reads +// these files is that one. The lab paid for this exactly once: a 0600 root-owned key copied in +// landed unreadable, `secret accept` failed with `permission denied`, and what depended on it +// crash-looped on material it never received. There is no shell in the image to chown it with. +// +// What goes through here is a module manifest and a store connection string. The connection is the +// same value the produced bundle already holds in the clear — a substrate names its own bootstrap +// credentials, and at genesis there is nowhere else for them to be — so this widens nothing. The +// file on the machine is removed at once, and the copy inside the container goes when the +// container does, which for the temporary control plane is step 10. +func (c controlPlane) carrying(ctx context.Context, name string, content []byte, remote string) error { + local := filepath.Join(os.TempDir(), name) + if err := os.WriteFile(local, content, 0o644); err != nil { + return fmt.Errorf("nowhere to stage %s before copying it into %s: %w", name, c.container, err) + } + defer os.Remove(local) + return c.carry(ctx, local, remote) +} + +func indent(s string) string { + if s == "" { + return "" + } + return " " + strings.ReplaceAll(s, "\n", "\n ") +} + +// mentions reports whether one of a listing's lines starts with this exact word. +// +// Line-and-word rather than a substring search, because these listings are columns and a +// substring match would find `registry` inside `registry-mirror` and report a module installed +// that is not. Every one of mesh-control's `list` verbs prints the name first on the line. +func mentions(listing, name string) bool { + for _, line := range strings.Split(listing, "\n") { + first, _, _ := strings.Cut(strings.TrimSpace(line), " ") + if first == name { + return true + } + } + return false +}