diff --git a/.gitignore b/.gitignore index 235d26c..d9cc9a8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ /mesh-host +/mesh-bootstrap /dist/ +# The placeholder `make bootstrap` moves aside while a saved image is embedded. Ignored so an +# interrupted release build cannot commit a twenty-megabyte tar by accident. +/internal/image/control-plane.tar.placeholder diff --git a/Makefile b/Makefile index ebe96e7..f65db19 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ LDFLAGS := -s -w -X main.builtFor=$(SYSTEM) -X main.version=$(VERSION) # a second file to arrive with it is not "copy it and run it". BUNDLE ?= -.PHONY: check test vet fmt build clean host +.PHONY: check test vet fmt build clean host hosts bootstrap packaging-test check: fmt vet test packaging-test build @@ -57,5 +57,44 @@ host: exit $$status @echo "built for $(SYSTEM) carrying $(BUNDLE)" +# The installer, carrying the control plane's image: +# make bootstrap IMAGE=mesh-control:v1.2.3 +# +# The image is BUILT ELSEWHERE and handed over — mesh-control's own `make image` — and embedded +# here at release time. Not built on the machine being bootstrapped, and not fetched: the forge +# that holds mesh-control's source runs on the mesh, so a bootstrap that had to fetch or build the +# control plane would need a mesh in order to raise one. Carrying it breaks that cycle, the same +# way carrying the bundle breaks the "copy it onto a machine and run it" one (novox/hq ADR 0005). +# +# The saved image occupies the embed slot for the length of one build and the placeholder goes +# back, exactly as `host:` does with the bundle. Nothing large is ever committed. +# +# IMAGE must be a NAME:TAG and not an id. The installer identifies the carried image by its tag, +# because an image id is the digest of the image's configuration and a runtime REWRITES that +# configuration as it loads — so the id in the archive is not the id the receiving machine will +# hold, and the tag is the only name that survives the transfer. Saving by id produces an archive +# with no tags at all, which the installer refuses; caught here instead, in front of the person who +# can fix it. +# +# BOOTSTRAP_OUT is where the binary is written, and it exists because something other than a person +# now builds this: the lab rebuilds every artifact it runs from source before a raise, into paths it +# chose (mesh-lab's src/rebuild.ts, and novox/hq 04-ISSUES/005 for why it does that at all). A +# caller that could not say where the output goes would have to copy it afterwards, which is one +# more step to forget. +BOOTSTRAP_OUT ?= mesh-bootstrap + +bootstrap: + @test -n "$(IMAGE)" || { echo "IMAGE= is required; an installer carrying no control-plane image cannot raise a mesh"; exit 1; } + @case "$(IMAGE)" in sha256:*) echo "IMAGE=$(IMAGE) is an image id. The installer identifies the carried image by its tag, because an id is the digest of a configuration that a runtime rewrites as it loads. Pass a name:tag"; exit 1;; esac + @docker image inspect "$(IMAGE)" >/dev/null 2>&1 || { echo "this machine does not hold $(IMAGE) — build it in mesh-control with 'make image'"; exit 1; } + @test -n "$$(docker image inspect --format '{{len .RepoTags}}' "$(IMAGE)" | grep -v '^0$$')" || { echo "$(IMAGE) has no repository tag, so the saved archive would carry no name the installer can ask a runtime about. Tag it first: docker tag $(IMAGE) mesh-control:"; exit 1; } + @cp internal/image/control-plane.tar internal/image/control-plane.tar.placeholder + @docker save --output internal/image/control-plane.tar "$(IMAGE)" + @CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=$(VERSION)" -o "$(BOOTSTRAP_OUT)" ./cmd/mesh-bootstrap; \ + status=$$?; \ + mv internal/image/control-plane.tar.placeholder internal/image/control-plane.tar; \ + exit $$status + @echo "built $(BOOTSTRAP_OUT) carrying $(IMAGE)" + clean: - rm -f mesh-host + rm -f mesh-host mesh-bootstrap diff --git a/cmd/mesh-bootstrap/main.go b/cmd/mesh-bootstrap/main.go new file mode 100644 index 0000000..633f668 --- /dev/null +++ b/cmd/mesh-bootstrap/main.go @@ -0,0 +1,283 @@ +// Command mesh-bootstrap brings a mesh into existence on a bare machine. +// +// Tier 0, beside `mesh-host` and not inside it. Bootstrapping is done by hand and it changes a +// machine, which is what tier 0 is (novox/hq 03-DESIGN/01-to-be/05-the-node-host.md) — but +// `mesh-host` states of itself that it connects to nothing and listens on nothing and that what it +// applies comes from a file, and that is the whole reason an always-running root daemon can be +// audited by reading one page. An installer that loads images and interrogates a control plane +// cannot be folded into it without making that sentence false. Same tier, same repository, +// different program. +// +// Genesis is a pivot (novox/hq ADR 0067). It raises a substrate whose control plane is named by the +// digest of its own configuration — legal exactly where nothing could have served an image — then +// enrols this machine, installs the registry module, pushes that image into it to get the manifest +// digest it has never had, reinstalls the control plane as an ordinary module pinned to it, and +// drops the temporary one. Without --catalog it stops after the substrate and says why. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/bootstrap" + "github.com/novox/mesh-host/internal/store" +) + +// version is stamped at build time. Unset in a development build, and said so rather than +// defaulted to something that looks like a release. +var version = "development build" + +const ( + defaultTemplate = "substrate.lock" + defaultOut = "/var/lib/mesh-host/substrate.lock" + defaultRegistry = "127.0.0.1:5000" + defaultHost = "/usr/local/bin/mesh-host" + defaultService = "mesh-host.service" +) + +const usage = `mesh-bootstrap — make a bare machine into a mesh + + bootstrap the ten steps below (the default) + version + + 1 preflight what has to be true before anything is changed + 2 load the control plane's image, carried in this installer + 3 bundle the substrate, named for this machine + 4 apply raise it + 5 verify it is up, and the control plane replies + 6 enrol this machine becomes the mesh's first node + 7 registry install the module that gives this mesh an image store + 8 publish push the control plane's image into it, for its first digest + 9 control reinstall the control plane as an ordinary module, pinned to that digest + 10 retire drop the temporary control plane; the host removes it + + --bundle the substrate template to build this machine's bundle from + (default ` + defaultTemplate + `) + --out where the produced bundle is written, for a person to read + (default ` + defaultOut + `) + --state where this node records what it has applied + (default ` + store.DefaultPath + `) + --catalog a checkout of the mesh's catalogue, holding the registry's and the + control plane's manifests. Without it this stops after step 5 + --node the name this machine is known by (default: its hostname) + --registry where this mesh keeps its own images (default ` + defaultRegistry + `) + every node pulls the control plane from this, so on a mesh of more + than one machine it must be an address the others can reach + --host the mesh-host binary on this machine (default ` + defaultHost + `) + --host-service the unit that supervises it (default ` + defaultService + `) + --host-in-background start the host unsupervised instead. It does not survive + a reboot. This is what a lab does and what no real machine should + --system which operating system this is; by default it is asked + --timeout how long any single probe may take (default 30s) + --wait how long a thing that is merely starting is given (default 3m) + --dry-run everything that does not change the machine + --json machine-readable output + +Genesis is a pivot: a temporary control plane installs the registry that makes it +permanent. The temporary one is called temp-mesh-control and the permanent one is +called mesh-control, so they are two containers with two owners and there is nothing +to hand over. + +Every step is idempotent: run it again after fixing whatever it named, and the steps +that already succeeded say so. +` + +func main() { + // Ctrl-C must stop the installer, not be swallowed by whatever it is waiting for — and it + // waits on pulls, on a runtime starting, and on a control plane opening its stores. + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + command, opts, jsonOut, err := parseArgs(os.Args[1:]) + if err == nil { + err = run(ctx, command, opts, jsonOut) + } + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-bootstrap: %v\n", err) + os.Exit(1) + } +} + +// parseArgs takes an optional subcommand first, then its flags. +// +// Parsed in a loop for the reason `mesh-host` records: the standard library stops at the FIRST +// non-flag argument, so a flag sitting after one is silently dropped and the command exits zero +// having ignored what it was asked. That fault has been paid for twice in this repository and is +// not being paid for a third time. +func parseArgs(args []string) (string, bootstrap.Options, bool, error) { + opts := bootstrap.Options{ + Template: defaultTemplate, + Out: defaultOut, + State: store.DefaultPath, + Registry: defaultRegistry, + Host: defaultHost, + // The machine's own name, because that is what a person already calls it and an installer + // inventing a different one would leave the mesh naming a machine nobody recognises. It is + // read here rather than inside the bootstrap so that --node overrides a fact rather than a + // default computed halfway through. + Node: hostname(), + HostService: defaultService, + // Longer than the host's 10s: these probes reach a container runtime that may be busy + // pulling, and a probe that times out on a working machine is a false refusal. + Timeout: 30 * time.Second, + // A socket-activated runtime queued behind the network, and a control plane running its + // first `initdb`-shaped wait, are both minutes rather than seconds. + Wait: 3 * time.Minute, + } + var jsonOut bool + + command := "bootstrap" + if len(args) > 0 && len(args[0]) > 0 && args[0][0] != '-' { + command = args[0] + args = args[1:] + } + + set := newFlagSet(&opts, &jsonOut) + var positionals []string + rest := args + for { + if err := set.Parse(rest); err != nil { + return "", opts, false, err + } + rest = set.Args() + if len(rest) == 0 { + break + } + positionals = append(positionals, rest[0]) + rest = rest[1:] + } + + // Refused rather than ignored: a mistyped argument that changes nothing and reports success is + // worse than an error, and this program's whole job is to change a machine. + if len(positionals) > 0 { + return "", opts, false, fmt.Errorf( + "unexpected argument %q — try `mesh-bootstrap help`", positionals[0]) + } + return command, opts, jsonOut, nil +} + +func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet { + set := flag.NewFlagSet("mesh-bootstrap", flag.ContinueOnError) + set.SetOutput(os.Stderr) + set.Usage = func() { fmt.Fprint(os.Stderr, usage) } + set.StringVar(&opts.Template, "bundle", opts.Template, "the substrate template to build from") + set.StringVar(&opts.Out, "out", opts.Out, "where the produced bundle is written") + set.StringVar(&opts.State, "state", opts.State, "where this node records what it has applied") + set.StringVar(&opts.Catalogue, "catalog", opts.Catalogue, + "a checkout of the mesh's catalogue; without it this stops after the substrate") + set.StringVar(&opts.Node, "node", opts.Node, "the name this machine is known by") + set.StringVar(&opts.Registry, "registry", opts.Registry, "where this mesh keeps its own images") + set.StringVar(&opts.Host, "host", opts.Host, "the mesh-host binary on this machine") + set.StringVar(&opts.HostService, "host-service", opts.HostService, "the unit that supervises it") + set.BoolVar(&opts.HostInBackground, "host-in-background", false, + "start the host unsupervised; it does not survive a reboot") + set.StringVar(&opts.System, "system", opts.System, "which operating system this is") + set.DurationVar(&opts.Timeout, "timeout", opts.Timeout, "how long any single probe may take") + set.DurationVar(&opts.Wait, "wait", opts.Wait, "how long something merely starting is given") + set.BoolVar(&opts.DryRun, "dry-run", false, "everything that does not change the machine") + set.BoolVar(jsonOut, "json", false, "machine-readable output") + return set +} + +func run(ctx context.Context, command string, opts bootstrap.Options, jsonOut bool) error { + switch command { + case "bootstrap": + say := func(line string) { + if !jsonOut { + fmt.Println(line) + } + } + result, err := bootstrap.Run(ctx, opts, bootstrap.Deps{ + Run: apply.ExecRunner, + Dial: dial, + Fetch: fetch, + }, say) + + // Printed whichever way it went. What the installer got through before it stopped is on + // the machine either way, and a report that only exists on success describes a machine + // nobody has (novox/hq ADR 0018). + if jsonOut { + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if encodeErr := encoder.Encode(result); encodeErr != nil && err == nil { + return encodeErr + } + } + return err + + case "version": + fmt.Println(version) + return nil + + case "help", "-h", "--help": + fmt.Fprint(os.Stderr, usage) + return nil + + default: + return fmt.Errorf("unknown command %q — try `mesh-bootstrap help`", command) + } +} + +// hostname is what this machine calls itself, or empty. +// +// Empty rather than a guess: a machine that cannot say its own name is one the installer must be +// told about, and `mesh-bootstrap-0` would be a name in the mesh's records that matches nothing +// anybody types anywhere else. The refusal happens at step 6, where the name is first needed. +func hostname() string { + name, err := os.Hostname() + if err != nil { + return "" + } + return name +} + +// fetch asks an HTTP endpoint and reports what it said. +// +// Plain HTTP, and only at the mesh's own registry: it is reached over the mesh's private network, +// which is already the encrypted and authenticated thing, and a second layer inside it would be +// certificates to issue and rotate for no property the first does not have (mesh-control's +// `internal/builder` pushes to it on the same reasoning). +// +// The body is read with a limit. What is asked for is a status and a short JSON answer, and a +// registry that answered with a gigabyte would otherwise be an installer that never returns. +func fetch(ctx context.Context, url string) (int, string, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return 0, "", err + } + response, err := http.DefaultClient.Do(request) + if err != nil { + return 0, "", err + } + defer response.Body.Close() + + said, err := io.ReadAll(io.LimitReader(response.Body, 1<<20)) + if err != nil { + return response.StatusCode, "", err + } + return response.StatusCode, string(said), nil +} + +// dial answers whether a TCP address responds. +// +// A connection rather than a ping or a name lookup: what has to work is a pull, and a pull opens a +// connection to exactly this address. A machine whose DNS resolves and whose route is missing +// passes a lookup and fails the thing that matters. +func dial(ctx context.Context, address string) error { + var dialer net.Dialer + conn, err := dialer.DialContext(ctx, "tcp", address) + if err != nil { + return err + } + return conn.Close() +} diff --git a/cmd/mesh-bootstrap/main_test.go b/cmd/mesh-bootstrap/main_test.go new file mode 100644 index 0000000..95815d1 --- /dev/null +++ b/cmd/mesh-bootstrap/main_test.go @@ -0,0 +1,164 @@ +package main + +import ( + "flag" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/bootstrap" + "github.com/novox/mesh-host/internal/store" +) + +// Argument handling gets tests for the reason `mesh-host` records: the standard library stops +// parsing at the first non-flag argument, so a flag sitting after one is silently dropped and the +// command exits zero having ignored what it was asked. Here that would mean `--dry-run` ignored on +// a program whose whole job is to change a machine. + +func TestBootstrapIsWhatItDoesWithNoCommand(t *testing.T) { + // Running the installer with nothing but flags must install, not print usage: the command is + // the reason the binary exists, and making somebody type its name twice buys nothing. + command, opts, _, err := parseArgs([]string{"--dry-run"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if command != "bootstrap" { + t.Errorf("command = %q, want bootstrap", command) + } + if !opts.DryRun { + t.Error("--dry-run before any subcommand was ignored") + } +} + +func TestFlagsAreReadWhereverTheySit(t *testing.T) { + for _, args := range [][]string{ + {"bootstrap", "--dry-run", "--bundle", "s.lock", "--json"}, + {"bootstrap", "--json", "--bundle=s.lock", "--dry-run"}, + {"--bundle", "s.lock", "--dry-run", "--json"}, + } { + _, opts, jsonOut, err := parseArgs(args) + if err != nil { + t.Errorf("%v: unexpected error: %v", args, err) + continue + } + if !opts.DryRun || !jsonOut || opts.Template != "s.lock" { + t.Errorf("%v parsed as dry-run=%v json=%v bundle=%q", + args, opts.DryRun, jsonOut, opts.Template) + } + } +} + +func TestAMistypedFlagIsRefusedNotIgnored(t *testing.T) { + // Asymmetric cost: an error is a moment's annoyance, and a silently dropped --dry-run is a + // machine changed by somebody who asked for it not to be. + if _, _, _, err := parseArgs([]string{"bootstrap", "--dry-runn"}); err == nil { + t.Fatal("a mistyped flag was accepted") + } +} + +func TestAnUnexpectedArgumentIsRefused(t *testing.T) { + if _, _, _, err := parseArgs([]string{"bootstrap", "substrate.lock"}); err == nil { + t.Fatal("a stray argument was ignored rather than refused — the bundle is --bundle") + } +} + +func TestTheDefaultsAreTheDocumentedOnes(t *testing.T) { + // The usage text is a promise. A default that drifts from what is printed is a small lie that + // costs somebody an afternoon in front of a machine that will not come up. + _, opts, jsonOut, err := parseArgs(nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if opts.Template != defaultTemplate { + t.Errorf("default bundle is %q; the usage text says %q", opts.Template, defaultTemplate) + } + if opts.Out != defaultOut { + t.Errorf("default out is %q; the usage text says %q", opts.Out, defaultOut) + } + if opts.State != store.DefaultPath { + t.Errorf("default state is %q; the host's own default is %q", opts.State, store.DefaultPath) + } + if opts.Timeout != 30*time.Second { + t.Errorf("default timeout is %s; the usage text says 30s", opts.Timeout) + } + if opts.Wait != 3*time.Minute { + t.Errorf("default wait is %s; the usage text says 3m", opts.Wait) + } + if opts.DryRun || jsonOut || opts.System != "" { + t.Error("something is on by default that the usage text describes as a flag") + } +} + +// Every flag the usage text promises must exist, and every flag that exists must be in the usage +// text. The two drifting apart is how a program acquires a feature nobody can find and a +// documented option that does nothing. +func TestTheUsageTextAndTheFlagsAgree(t *testing.T) { + var opts bootstrap.Options + var jsonOut bool + set := newFlagSet(&opts, &jsonOut) + + declared := map[string]bool{} + set.VisitAll(func(f *flag.Flag) { declared[f.Name] = true }) + + for name := range declared { + if !strings.Contains(usage, "--"+name) { + t.Errorf("--%s exists and the usage text does not mention it", name) + } + } + for _, promised := range []string{ + "bundle", "out", "state", "system", "timeout", "wait", "dry-run", "json", + "catalog", "node", "registry", "host", "host-service", "host-in-background", + } { + if !declared[promised] { + t.Errorf("the usage text promises --%s and no such flag exists", promised) + } + } +} + +// The pivot's defaults are the documented ones too, and the one that has no default is the one +// that decides whether the pivot happens at all. +func TestThePivotsDefaultsAreTheDocumentedOnes(t *testing.T) { + _, opts, _, err := parseArgs(nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if opts.Registry != defaultRegistry { + t.Errorf("default registry is %q; the usage text says %q", opts.Registry, defaultRegistry) + } + if opts.Host != defaultHost { + t.Errorf("default host binary is %q; the usage text says %q", opts.Host, defaultHost) + } + if opts.HostService != defaultService { + t.Errorf("default host service is %q; the usage text says %q", + opts.HostService, defaultService) + } + if opts.HostInBackground { + t.Error("the host is started unsupervised by default, and no real machine should") + } + // **No default, deliberately.** A catalogue this installer went looking for on its own would + // be a checkout somebody else made, at whatever commit they left it on — and it decides which + // image the mesh's control plane is pinned to for ever after. + if opts.Catalogue != "" { + t.Errorf("--catalog defaults to %q; without one the installer stops at the substrate", + opts.Catalogue) + } + // The machine's own name, because that is what a person already calls it. + if opts.Node == "" { + t.Error("no default node name; this machine can say what it is called") + } +} + +// --node overrides the machine's own name rather than being ignored because a default was already +// computed. The name is what the mesh's records, tokens, assignments and pushes all name. +func TestTheNodeNameCanBeSaid(t *testing.T) { + _, opts, _, err := parseArgs([]string{"--node", "anchor", "--catalog", "/somewhere"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if opts.Node != "anchor" { + t.Errorf("--node anchor parsed as %q", opts.Node) + } + if opts.Catalogue != "/somewhere" { + t.Errorf("--catalog parsed as %q", opts.Catalogue) + } +} diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 6b2c65d..baba8d4 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -1154,6 +1154,15 @@ func ensureImage(ctx context.Context, cri, image string, run Runner) error { if _, err := run(ctx, cri, "image", "inspect", image); err == nil { return nil } + // An image named by the digest of its own configuration is one this machine was supposed to + // already hold — built here, or handed over. There is no registry that answers for it, so + // pulling would fail somewhere that names a network problem instead of a missing image. + if strings.HasPrefix(image, "sha256:") { + return fmt.Errorf( + "image %s is not on this machine, and an image named by its own digest cannot be "+ + "fetched: nothing serves it. Build it here, or load it, before applying this", + image) + } if _, err := run(ctx, cri, "pull", image); err != nil { return fmt.Errorf("pulling image %s: %w", image, err) } diff --git a/internal/bootstrap/apply.go b/internal/bootstrap/apply.go new file mode 100644 index 0000000..6f8e7c0 --- /dev/null +++ b/internal/bootstrap/apply.go @@ -0,0 +1,134 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// Runner is the same runner every applier in this repository takes. +type Runner = apply.Runner + +// ApplyBundle raises the substrate, through the host's own apply. +// +// **This calls `internal/apply` rather than running the `mesh-host` binary**, and that is worth +// stating because shelling out would have been easier. The installer and the host must apply a +// declaration identically — same removal pass, same read-backs, same refusal model, same record of +// what this machine now owns — and two code paths that must behave the same are two code paths +// that will not. The `mesh-host` binary is also not guaranteed to be on a machine this program is +// raising, which would make the installer depend on the thing it installs. +// +// It applies under `store.OriginCarried`, which is the same origin `mesh-host reconcile` uses and +// is not a detail: what the substrate raised must be invisible to the removal pass of a +// declaration that later arrives from the control plane, or the first thing the mesh tells this +// node would tear down the mesh (novox/hq 04-ISSUES/010). +// +// What it does not do is the host's own lifecycle bookkeeping — recording a known-good version, +// clearing the launcher's start counter. Those are facts about a running `mesh-host`, and this is +// not one. +func ApplyBundle(ctx context.Context, o Options, sys system.System, d *declaration.Declaration, + run Runner, say func(string)) (apply.Report, error) { + + // Refuse a shape this host cannot apply before anything is applied, exactly as `mesh-host` + // does: finding out half way through is the half-configured machine tier 0 exists to prevent. + if err := system.Check(sys, d); err != nil { + return apply.Report{}, err + } + + known, err := store.Load(o.State) + if err != nil { + return apply.Report{}, err + } + + report, updated, applyErr := apply.Apply(ctx, sys, d, known, store.OriginCarried, run, + func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed) + + // Saved whichever way it went, for the reason `mesh-host` gives: what was applied before a + // failure is on the machine either way, and a host that did not record it would believe it + // owns less than it does and leave that behind for ever. + if saveErr := store.Save(o.State, updated); saveErr != nil { + if applyErr != nil { + return report, fmt.Errorf("%w\n\nand this node's state could not be saved: %v", + applyErr, saveErr) + } + return report, saveErr + } + if applyErr != nil { + return report, fmt.Errorf("%w\n\nThe machine is in whatever state that left it. Fix what "+ + "is named above and run this again — every step is idempotent, and the ones that "+ + "already succeeded will say so", applyErr) + } + return report, nil +} + +// refuseSealed is what happens when a bundle contains a file the mesh sealed to this node. +// +// It cannot happen and it is refused with a sentence rather than a nil dereference. A sealing key +// is generated at enrolment (`internal/identity`), and enrolment is something that happens on a +// mesh — which is the thing this program is raising. A substrate bundle carrying a sealed file +// would be a bundle written for a node that has already joined. +func refuseSealed(string) ([]byte, error) { + return nil, errors.New( + "this bundle contains a file sealed to a node's key, and a machine that has not enrolled " + + "has no such key. A substrate is applied before any mesh exists, so it can carry no " + + "secret the mesh sealed") +} + +// WorkOutSystem decides which half of the host applies things on this machine, and proves it. +// +// `mesh-host` pins this at link time because it is built for one operating system and refuses to +// touch a machine without knowing which (novox/hq ADR 0005). An installer run by hand has no +// link-time to pin it at, so it asks — but it does not guess: every system already knows how to +// prove it is the one it claims to be, by asking its package database about a package that is +// certainly there. Exactly one may answer. +// +// A machine where none answers is refused with what each of them said, because "unsupported +// system" is a sentence nobody can act on and "pacman does not answer here" is. +func WorkOutSystem(ctx context.Context, run Runner, named string) (system.System, error) { + if strings.TrimSpace(named) != "" { + chosen, err := system.For(named) + if err != nil { + return nil, err + } + if err := chosen.Confirm(ctx, run); err != nil { + return nil, fmt.Errorf("--system %s was given, and this machine says otherwise: %w", + named, err) + } + return chosen, nil + } + + var answered []system.System + var refusals []string + for _, candidate := range system.All() { + if err := candidate.Confirm(ctx, run); err != nil { + refusals = append(refusals, fmt.Sprintf(" %s: %v", candidate.Name(), err)) + continue + } + answered = append(answered, candidate) + } + + switch len(answered) { + case 1: + return answered[0], nil + case 0: + return nil, fmt.Errorf( + "this machine is none of the systems this installer knows how to change, so nothing "+ + "was attempted:\n%s\nName one with --system if it is really one of them and its "+ + "package database is merely unwell", strings.Join(refusals, "\n")) + default: + var names []string + for _, s := range answered { + names = append(names, s.Name()) + } + return nil, fmt.Errorf( + "this machine answers as %s at once, and the installer must not choose between them: "+ + "package names and unit names differ, and picking wrong misconfigures the machine "+ + "quietly. Say which with --system", strings.Join(names, " and ")) + } +} diff --git a/internal/bootstrap/apply_test.go b/internal/bootstrap/apply_test.go new file mode 100644 index 0000000..7a3c8ea --- /dev/null +++ b/internal/bootstrap/apply_test.go @@ -0,0 +1,91 @@ +package bootstrap + +import ( + "context" + "errors" + "strings" + "testing" +) + +// `mesh-host` is built for one operating system and pins it at link time. An installer run by hand +// has no link time, so it asks — and it does not guess: each system already knows how to prove it +// is the one it claims to be, by asking its package database about a package that is certainly +// there. Getting this wrong installs with the wrong package manager and the wrong unit names. + +func TestTheMachineIsAskedWhichSystemItIs(t *testing.T) { + // Only pacman answers, so this is the arch host and nothing had to be told so. + onlyPacman := func(_ context.Context, name string, _ ...string) (string, error) { + if name == "pacman" { + return "pacman 7.0.0-1\n", nil + } + return "", errors.New("command not found") + } + + chosen, err := WorkOutSystem(context.Background(), onlyPacman, "") + if err != nil { + t.Fatal(err) + } + if chosen.Name() != "arch" { + t.Errorf("this machine was worked out to be %q", chosen.Name()) + } +} + +// A machine that is none of them is refused with what each of them said. "Unsupported system" is +// a sentence nobody can act on; "pacman does not answer here" is. +func TestAMachineThatIsNoneOfThemIsRefusedWithWhatEachSaid(t *testing.T) { + nothing := func(context.Context, string, ...string) (string, error) { + return "", errors.New("command not found") + } + + _, err := WorkOutSystem(context.Background(), nothing, "") + if err == nil { + t.Fatal("a machine that answers as no known system was accepted") + } + for _, wanted := range []string{"arch:", "alpine:", "--system"} { + if !strings.Contains(err.Error(), wanted) { + t.Errorf("the refusal does not mention %q:\n%v", wanted, err) + } + } +} + +// And a machine that was TOLD what it is still has to prove it. Installing the arch half of the +// host on Alpine must say so once, at the start, rather than failing later inside pacman. +func TestASystemThatWasNamedIsStillProved(t *testing.T) { + onlyApk := func(_ context.Context, name string, _ ...string) (string, error) { + if name == "apk" { + return "apk-tools-2.14.0\n", nil + } + return "", errors.New("command not found") + } + + if _, err := WorkOutSystem(context.Background(), onlyApk, "arch"); err == nil { + t.Fatal("--system arch was believed on a machine where pacman does not answer") + } + + if _, err := WorkOutSystem(context.Background(), onlyApk, "alpine"); err != nil { + t.Errorf("--system alpine was refused on a machine where apk answers: %v", err) + } +} + +func TestASystemNobodyHasBuiltIsRefusedByName(t *testing.T) { + anything := func(context.Context, string, ...string) (string, error) { return "", nil } + _, err := WorkOutSystem(context.Background(), anything, "debian") + if err == nil { + t.Fatal("--system debian was accepted, and no debian host is built") + } + if !strings.Contains(err.Error(), "arch") { + t.Errorf("the refusal does not say which systems exist: %v", err) + } +} + +// A substrate is applied before any mesh exists, so it can carry no secret the mesh sealed — there +// is no key to open one with. Refused with a sentence rather than a nil dereference. +func TestASealedFileInASubstrateIsRefusedWithAReason(t *testing.T) { + _, err := refuseSealed("anything") + if err == nil { + t.Fatal("a sealed file in a substrate bundle was accepted") + } + if !strings.Contains(err.Error(), "has not enrolled") { + t.Errorf("the refusal does not say why there is no key: %v", err) + } +} diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go new file mode 100644 index 0000000..ae31339 --- /dev/null +++ b/internal/bootstrap/bootstrap.go @@ -0,0 +1,477 @@ +// Package bootstrap brings a mesh into existence on a bare machine. +// +// **Why this is a program at all.** Until now the only complete written-down copy of the +// first-node procedure was an integration test in the lab — `whole-mesh-full.test.ts` — which is +// why every gap in it kept being found late and by accident: an install procedure that lives as a +// test fixture is exercised by whoever is writing tests, never by whoever is installing. This is +// that procedure, made into the thing it always was. +// +// **Tier 0, and a separate binary.** Bootstrapping is done by hand and it changes a machine, so by +// novox/hq 03-DESIGN/01-to-be/05-the-node-host.md it is tier 0 and belongs beside the host. It is +// not a `mesh-host` subcommand, because `mesh-host` says of itself that it connects to nothing and +// listens on nothing and that what it applies comes from a file — a property that is what makes an +// always-running root daemon auditable, and that must stay literally true. This program pulls +// images and asks a running control plane questions. Same tier, same repository, different binary. +// +// **No registry is required for the mesh's own image, and no source either.** The control plane +// exists in no registry by design, and the forge that holds its source runs on the mesh — so a +// bootstrap that fetched or built it would need a mesh in order to raise a mesh. The installer +// carries the image (`internal/image`) and names it by its image id: the sha256 of its own +// configuration, which is exact, unforgeable, and needs nothing to have served it (novox/hq +// ADR 0006, and `internal/declaration`'s checkImage). Third-party images keep their upstream +// `name@sha256:` references and are pulled from the internet like anything else. +// +// **And that id is read back from the machine, never predicted from the archive.** The digest of a +// configuration is not portable: a runtime rewrites the configuration as it loads, so the same +// bytes are held under a different name on the machine that receives them than on the one that +// saved them. The archive is identified by its TAG, which does survive the transfer, and the id +// the bundle names is whatever the runtime answers for that tag afterwards. See Load, which +// carries the measurement. +package bootstrap + +import ( + "context" + "fmt" + "strings" + "time" +) + +// Step names one stage. A failure says which one, because "the bootstrap failed" is a sentence +// nobody can act on and this will be run over and over by somebody getting a machine working. +type Step string + +const ( + StepPreflight Step = "preflight" + StepLoad Step = "load" + StepBundle Step = "bundle" + StepApply Step = "apply" + StepVerify Step = "verify" + StepEnrol Step = "enrol" + StepRegistry Step = "registry" + StepPublish Step = "publish" + StepControlPlane Step = "control-plane" + StepRetire Step = "retire" +) + +// Steps in the order they happen, so a failure can say "step 2 of 10". +// +// The first five make a machine; the last five make a mesh that can maintain itself. They are one +// program because they are one procedure — the whole reason the pivot exists is that steps 7 to 9 +// cannot happen without steps 1 to 5, and steps 1 to 5 leave something that cannot be upgraded +// without steps 7 to 9 (novox/hq ADR 0067). +var Steps = []Step{ + StepPreflight, StepLoad, StepBundle, StepApply, StepVerify, + StepEnrol, StepRegistry, StepPublish, StepControlPlane, StepRetire, +} + +// Error is a failure, named by the step it happened in. +type Error struct { + Step Step + Err error +} + +func (e *Error) Error() string { + at := 0 + for i, s := range Steps { + if s == e.Step { + at = i + 1 + } + } + return fmt.Sprintf("step %d of %d, %s: %v", at, len(Steps), e.Step, e.Err) +} + +func (e *Error) Unwrap() error { return e.Err } + +func failed(step Step, err error) error { + if err == nil { + return nil + } + return &Error{Step: step, Err: err} +} + +// Options are the things that differ between machines. +type Options struct { + // Template is the substrate bundle this machine's own bundle is made from. + Template string + // Out is where the produced bundle is written, so a person can read what was applied. + Out string + // State is where the host records what it has applied here — the same file `mesh-host` reads, + // because what this raises the host must afterwards own. + State string + // System is which half of the host applies things. Empty means ask the machine. + System string + // DryRun does everything that does not change the machine. + DryRun bool + // Timeout bounds any single probe. + Timeout time.Duration + // Wait is how long something that is merely starting is given: a socket-activated container + // runtime, a control plane opening its stores. + Wait time.Duration + + // Node is the name this machine is known by in the mesh. Everything after the substrate names + // it: the record, the token, the assignment, the push. + Node string + + // Catalogue is a checkout of the mesh's catalogue repository, which is where the registry's and + // the control plane's manifests are read from. Empty stops the installer after the substrate: + // there is no pivot without manifests, and pretending otherwise would leave a machine that + // looks installed and cannot upgrade itself. + Catalogue string + + // Registry is where this mesh's own images live, as this machine reaches it. Every node will + // pull the control plane from what this says, so on a mesh of more than one machine it must be + // an address the others can reach. + Registry string + + // Host is the `mesh-host` binary on this machine — the program that enrols and then holds the + // machine to what the mesh says. The installer runs it; it does not contain it. + Host string + // HostService is the unit that supervises it. Started and enabled, never written: what a unit + // says is a packaging decision, and an installer inventing one would put a file on the machine + // that whatever installed the host will disagree with. + HostService string + // HostInBackground starts the host unsupervised instead, which is what the lab does and what no + // real machine should do — it does not survive a reboot. + HostInBackground bool +} + +// pivots reports whether this run goes past the substrate. +func (o Options) pivots() bool { return strings.TrimSpace(o.Catalogue) != "" } + +// Deps are the ways this program reaches outside itself. Injected so the whole of it can be +// tested without a container runtime, a network, or a machine to break — the same reason +// `internal/apply` takes a Runner (novox/hq ADR 0017). +type Deps struct { + // Run executes a command. apply.ExecRunner in production. + Run Runner + // Dial reports whether a TCP address answers, for "can this machine reach the registries the + // bundle names". + Dial func(ctx context.Context, address string) error + // Fetch asks an HTTP endpoint and reports what it said. Used only against the mesh's own + // registry: a container that is up is not a registry that serves, and `/v2/` is the one + // question whose answer means it is. + Fetch func(ctx context.Context, url string) (int, string, error) +} + +// Result is what the bootstrap did, in the shape `--json` prints. +type Result struct { + System string `json:"system"` + DryRun bool `json:"dry-run,omitempty"` + + // Image is what THIS MACHINE'S RUNTIME holds the control plane as, read back from it — and + // what the produced bundle names it by. + Image string `json:"image,omitempty"` + // ImageArchive is what the carried archive calls the same image. Reported because it is + // routinely a DIFFERENT id: a runtime rewrites an image's configuration as it loads, and an id + // is that configuration's digest. Never what the bundle names. + ImageArchive string `json:"image-in-archive,omitempty"` + // ImageTag is the name the runtime was asked by, which is how the id above was obtained. + ImageTag string `json:"image-tag,omitempty"` + // ImageTags is everything the image was called when it was saved. + ImageTags []string `json:"image-tags,omitempty"` + // ImageHeld is true when the machine already held it and nothing was loaded. + ImageHeld bool `json:"image-already-held,omitempty"` + // ImagePredicted is true when Image is the archive's id because nothing was loaded — a dry run + // only, and the reason a dry run does not claim to know what would be applied. + ImagePredicted bool `json:"image-id-is-a-prediction,omitempty"` + + // Bundle is where the produced bundle was written, and what was done to produce it. + Bundle string `json:"bundle,omitempty"` + BundleWas string `json:"bundle-replaced,omitempty"` + BundlePlaces int `json:"bundle-places,omitempty"` + BundleWrote bool `json:"bundle-written,omitempty"` + + // Applied is how many resources the apply reported on, and whether any of them moved. + Applied int `json:"applied,omitempty"` + Changed bool `json:"changed,omitempty"` + + // Running is the substrate's containers, confirmed up. + Running []string `json:"running,omitempty"` + // Answered is what the temporary control plane said back — not merely that it is up. + Answered string `json:"temporary-control-plane,omitempty"` + // Temporary is what the substrate's control plane is called, which is not what the module's is. + Temporary string `json:"temporary-container,omitempty"` + + // Node is this machine's name in the mesh, and how it came to be enrolled and heard from. + Node string `json:"node,omitempty"` + NodeAdded bool `json:"node-record-created,omitempty"` + Enrolled bool `json:"enrolled-now,omitempty"` + Agent string `json:"host-agent,omitempty"` + + // Registry is the mesh's own artifact store, once it answers. + Registry string `json:"registry,omitempty"` + RegistryReplied int `json:"registry-replied,omitempty"` + RegistryKnown bool `json:"registry-already-registered,omitempty"` + RegistryRunning string `json:"registry-container,omitempty"` + PublishedAs string `json:"control-plane-image,omitempty"` + PublishedAlready bool `json:"control-plane-image-already-published,omitempty"` + + // Permanent is the control plane as an ordinary module. + Permanent string `json:"permanent-container,omitempty"` + PermanentAnswered string `json:"permanent-control-plane,omitempty"` + StoresDelivered []string `json:"stores-delivered,omitempty"` + TemporaryRetired bool `json:"temporary-retired,omitempty"` + TemporaryWasGone bool `json:"temporary-was-already-gone,omitempty"` + TemporaryRemovedAt int `json:"resources-removed,omitempty"` + + // Stopped names why a run went no further. Empty on a run that pivoted. + Stopped string `json:"stopped,omitempty"` +} + +// Run performs the bootstrap, saying what it is doing as it goes. +// +// Every step is idempotent, and every step says whether it found something or changed it. That is +// not politeness: this program is run repeatedly while somebody gets a machine working, and a step +// that cannot tell "already done" from "just done" makes the second run indistinguishable from the +// first — which is how a person stops believing any of it. +// +// It does not retry. A pull that failed for a reason that goes away by itself is real, and the +// answer to it is to run this again: re-running is the retry, and it is one a person chooses after +// reading which step failed and why. +// +// **Genesis is a pivot** (novox/hq ADR 0067). Steps 1 to 5 raise a substrate whose control plane is +// named by the digest of its own configuration, because nothing has ever served that image and +// nothing could have. Steps 6 to 10 turn that into a mesh that can maintain itself: this machine +// enrols, the registry module is installed, the carried image is pushed INTO that registry — which +// gives it a manifest digest, its first — and the control plane is reinstalled as an ordinary +// module pinned to it. The temporary one is then dropped from the bundle and the host removes it. +// +// **What makes the last part expressible is a name.** The substrate's control plane is called +// `temp-mesh-control` and the module's is called `mesh-control`. Two containers, two owners: +// nothing is handed over, nothing has to stop being owned without being destroyed, and destruction +// by omission is the right end for something named "temp". +// +// Without --catalog it stops after step 5 and says so, because there are no manifests to install +// and a machine that looks installed and cannot upgrade itself is worse than one that stopped. +// +// **What an interruption leaves, at every step, and how a re-run continues.** This matters more +// here than anywhere else in the repository, because a machine left without a control plane cannot +// be fixed remotely — so no step may leave one: +// +// 1–3 nothing on the machine but a written file. Re-run: the bundle is produced again. +// 4 a partly-raised substrate, recorded in the state file. Re-run: apply converges the rest. +// 5 everything up; something did not answer yet. Re-run: it is asked again. +// 6 a node record and possibly a spent token. Re-run: `node list` finds the record, the +// identity file says whether this machine enrolled, and a fresh token is issued if not. +// 7 the registry registered, assigned, maybe not applied. Re-run: registered again (an +// upsert), pushed again, waited for again. The temporary control plane is untouched. +// 8 the image pushed and the digest unread. Re-run: the registry is asked what it holds and +// the answer is the same digest; nothing is pushed twice. +// 9 the module registered and the container not yet up, OR up beside the temporary one. Both +// are working states: the mesh has a control plane throughout. Re-run: continues. +// 10 the bundle rewritten and the container still there. Re-run: the bundle already omits it +// and the apply removes it; a bundle that already omits it reports "already dropped". +// +// The only step that cannot be undone by re-running is enrolment, and that is refused rather than +// repeated: a second identity is one the mesh does not know, and the mesh believes the first. +func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, error) { + if say == nil { + say = func(string) {} + } + result := Result{DryRun: o.DryRun} + + // ---- 1. preflight ------------------------------------------------------------------- + say("preflight — what has to be true before anything is changed") + template, err := Preflight(ctx, o, d, say) + if err != nil { + return result, failed(StepPreflight, err) + } + + // Which half of the host applies things here. Asked of the machine and proved, because + // `mesh-host` pins this at link time and an installer run by hand has no link time. + sys, err := WorkOutSystem(ctx, d.Run, o.System) + if err != nil { + return result, failed(StepPreflight, err) + } + result.System = sys.Name() + say(" system " + sys.Name()) + + // ---- 2. load ------------------------------------------------------------------------ + say("load — the control plane's image, carried in this installer") + loaded, err := Load(ctx, d.Run, o.DryRun, say) + if err != nil { + return result, failed(StepLoad, err) + } + result.Image, result.ImageTags, result.ImageHeld = loaded.ID, loaded.Tags, loaded.Held + result.ImageArchive, result.ImageTag = loaded.Archive, loaded.Tag + result.ImagePredicted = loaded.Predicted + + // ---- 3. bundle ---------------------------------------------------------------------- + say("bundle — what this machine will be asked to be") + rewritten, err := Rewrite(template, loaded.ID) + if err != nil { + return result, failed(StepBundle, err) + } + result.BundleWas, result.BundlePlaces, result.Bundle = rewritten.Was, rewritten.Places, o.Out + result.Temporary = rewritten.TempName + if rewritten.Renamed { + say(fmt.Sprintf(" control plane %s, renamed from %s", + rewritten.TempName, rewritten.WasCalled)) + say(" the permanent one is a module and takes the plain name; " + + "this one is dropped at the end") + } else { + say(" control plane " + rewritten.TempName + " — the template already named it that") + } + if rewritten.Changed { + say(fmt.Sprintf(" its image %s", rewritten.Now)) + say(fmt.Sprintf(" replacing %s, named in %d place(s)", + rewritten.Was, rewritten.Places)) + } else { + say(fmt.Sprintf(" its image %s — the template already named it, nothing rewritten", + rewritten.Now)) + } + if loaded.Predicted { + say(" UNCONFIRMED that id is the archive's and no runtime has been asked. A real " + + "run reads it back.") + } + for _, kept := range rewritten.Kept { + say(" left alone " + kept) + } + if rewritten.BrokerAddress != "" { + // Said every time, and never changed. Every enrolment token this mesh issues will tell a + // joining node to dial this address, and a wrong one is silent until the second node fails + // to come back. The installer does not know this machine's address and will not invent it. + say(" nodes will dial " + rewritten.BrokerAddress + + " — check this is an address other machines can reach") + } + + if o.DryRun { + // Nothing is written, exactly as `mesh-host --dry-run` reads a declaration and refuses it + // if wrong while changing nothing. The bundle has been produced and re-parsed in memory, + // which is everything that could be checked without touching the machine; what is left is + // loading, applying and asking the result questions, and none of those can be answered by + // not doing them. + say(fmt.Sprintf(" would write %s (%d resources)", o.Out, rewritten.Resources)) + if o.pivots() { + // Named rather than attempted. Everything from step 6 on is a conversation with a + // control plane that a dry run has not raised, so there is nothing to ask and nothing + // honest to report about the answers. + say(" would then enrol " + o.Node + ", install the registry from " + + o.Catalogue + ", push the control plane's image into it,") + say(" reinstall the control plane as a module, and drop " + + rewritten.TempName) + } + result.Stopped = "dry run: the bundle was produced and checked, and nothing was written, " + + "loaded or applied" + if loaded.Predicted { + result.Stopped += ". The image id in it is the archive's own and is not necessarily " + + "the one this machine would hold — a runtime rewrites an image's configuration as " + + "it loads, and the id is that configuration's digest" + } + say("\n" + result.Stopped) + return result, nil + } + + // **Nothing predicted is ever written down.** The line above is the only path on which + // `loaded.ID` can be the archive's id, and it returns. Asserted here rather than left to the + // reader, because what would follow is a bundle naming an image this machine does not hold — + // and nothing serves an image named by the digest of its own configuration, so it would fail + // inside a pull that cannot succeed, three steps from the cause. + if loaded.Predicted { + return result, failed(StepBundle, fmt.Errorf( + "the control plane's image id was never confirmed against this machine's runtime, and "+ + "the bundle was about to be written with it. This is a fault in the installer, not "+ + "in the machine")) + } + + if err := writeBundleFile(o.Out, rewritten.Bundle); err != nil { + return result, failed(StepBundle, err) + } + result.BundleWrote = true + say(fmt.Sprintf(" wrote %s (%d resources) — read it, this is what is applied", + o.Out, rewritten.Resources)) + + // ---- 4. apply ----------------------------------------------------------------------- + say("apply — raising the substrate") + report, err := ApplyBundle(ctx, o, sys, rewritten.Declaration, d.Run, say) + result.Applied, result.Changed = len(report.Outcomes), report.Changed() + if err != nil { + return result, failed(StepApply, err) + } + if report.Changed() { + say(fmt.Sprintf(" applied %d resource(s)", len(report.Outcomes))) + } else { + say(fmt.Sprintf(" already matches %d resource(s) checked, nothing moved", + len(report.Outcomes))) + } + + // ---- 5. verify ---------------------------------------------------------------------- + say("verify — the substrate is up, and the control plane replies") + verified, err := Verify(ctx, rewritten.Declaration, d.Run, o.Timeout, o.Wait, say) + result.Running, result.Answered = verified.Running, verified.Answered + if err != nil { + return result, failed(StepVerify, err) + } + + if !o.pivots() { + // Stopped, and said plainly. What has been raised works and cannot be upgraded: its + // control plane is named by an image id, which no registry serves, so nothing can ever + // replace it with a newer one. That is the whole of what the pivot fixes, and it needs + // manifests, and manifests come from a checkout somebody has to point this at. + result.Stopped = "no --catalog was given, so this stopped at the substrate. " + + "The control plane is named by the digest of its own configuration and no registry " + + "serves it, so this mesh cannot yet upgrade itself. Run again with " + + "--catalog to finish the pivot; every step " + + "above will say it is already done" + say("\nthis machine is a mesh of one node, with nothing joined to it yet.") + say(result.Stopped) + return result, nil + } + + // ---- 6. enrol ------------------------------------------------------------------------- + // + // From here on the mesh is being told things, and the way to tell it anything is to run its + // own binary inside its own container. `temporary` is the substrate's control plane; the + // module's is a different container with a different name and does not exist yet. + temporary := controlPlane{container: rewritten.TempName, run: d.Run, timeout: o.Timeout} + + say("enrol — this machine joins the mesh it is running") + enrolled, err := Enrol(ctx, o, sys, temporary, say) + result.Node, result.NodeAdded, result.Enrolled = enrolled.Node, enrolled.Added, enrolled.Joined + result.Agent = enrolled.Agent + if err != nil { + return result, failed(StepEnrol, err) + } + + // ---- 7. registry ---------------------------------------------------------------------- + say("registry — somewhere for this mesh to keep its own images") + registry, err := InstallRegistry(ctx, o, d, temporary, say) + result.Registry, result.RegistryRunning = registry.Address, registry.Container + result.RegistryKnown, result.RegistryReplied = registry.Known, registry.Answered + if err != nil { + return result, failed(StepRegistry, err) + } + + // ---- 8. publish ----------------------------------------------------------------------- + say("publish — the control plane's image gets its first manifest digest") + published, err := PublishControlPlane(ctx, o, d, loaded.ID, say) + result.PublishedAs, result.PublishedAlready = published.Reference, published.Already + if err != nil { + return result, failed(StepPublish, err) + } + + // ---- 9. control plane ----------------------------------------------------------------- + say("control plane — installed as an ordinary module, pinned to that digest") + permanent, err := InstallControlPlane(ctx, o, d, temporary, rewritten.Declaration, + published.Reference, say) + result.Permanent, result.PermanentAnswered = permanent.Container, permanent.Answered + result.StoresDelivered = permanent.Delivered + if err != nil { + return result, failed(StepControlPlane, err) + } + + // ---- 10. retire ----------------------------------------------------------------------- + say("retire — the temporary control plane is dropped from the bundle") + retired, err := RetireTheTemporaryControlPlane(ctx, o, sys, rewritten.Bundle, d.Run, say) + result.TemporaryRetired = retired.Gone || retired.Already + result.TemporaryWasGone, result.TemporaryRemovedAt = retired.Already, retired.Removed + if err != nil { + return result, failed(StepRetire, err) + } + + say("\nthis machine is a mesh of one node, and the control plane it runs is a module " + + "pinned to an image its own registry serves.") + say("what remains is somebody else's: adding nodes, and assigning what they should run.") + + return result, nil +} diff --git a/internal/bootstrap/control.go b/internal/bootstrap/control.go new file mode 100644 index 0000000..6f6d937 --- /dev/null +++ b/internal/bootstrap/control.go @@ -0,0 +1,410 @@ +package bootstrap + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/novox/mesh-host/internal/declaration" +) + +// storeFileSuffix is how a manifest asks for a store connection in a file rather than in the +// environment. +// +// `MESH_STORE_` is what the control plane reads (mesh-control's `internal/store`.Variable) +// and putting a password in a container's environment puts it in `docker inspect` for ever. So a +// module manifest names a file per context and points at it with `…_FILE`; the mesh seals the value +// into that file on the machine, and nothing but the process reads it. +const storeFileSuffix = "_FILE" + +// storeVariablePrefix is the front of the same names. +const storeVariablePrefix = "MESH_STORE_" + +// Permanent is what step 9 did. +type Permanent struct { + Installed + // Container is what the module calls its container, confirmed running. + Container string + // Image is what it is pinned to — the digest step 8's push produced. + Image string + // Delivered is every store connection accepted into it, by secret name. + Delivered []string + // Answered is what the permanent control plane said back. + Answered string +} + +// InstallControlPlane makes the control plane an ordinary module. +// +// **The host performs the replacement, not the control plane** (novox/hq ADR 0067). The temporary +// control plane composes a declaration naming the registry-pinned image, publishes it, and this +// node's host creates the container. Nothing is asked to replace itself while running, which is +// what makes the whole thing expressible: the container being created is called `mesh-control` and +// the one composing it is called `temp-mesh-control`, so there are two of them and neither is in +// the other's way. +// +// **The store connections are the substrate's, made at genesis, and the mesh cannot invent them.** +// Every other secret in a mesh is one the mesh made; these existed before the mesh did — they are +// the credentials the substrate bundle created the databases with. Generating replacements would +// put thirty-two random bytes where a working connection string has to be, and the control plane +// would come up unable to open a single context. So they go in through `secret accept`, which is +// exactly the path for a value the mesh must carry and could not have invented — and they are read +// out of the bundle this installer produced rather than reconstructed, because the bundle is what +// created them and a second opinion about what a DSN should say is a second chance to be wrong. +func InstallControlPlane(ctx context.Context, o Options, d Deps, control controlPlane, + substrate *declaration.Declaration, image string, say func(string)) (Permanent, error) { + + out := Permanent{Image: image} + + manifest, err := readManifest(o.Catalogue, ControlPlaneModule) + if err != nil { + return out, fmt.Errorf( + "%w\n"+ + "This is the manifest that makes the control plane an ordinary module. Without it "+ + "the machine keeps the temporary control plane the substrate raised, which works "+ + "and cannot be upgraded — so the install stops here rather than pretending to "+ + "have pivoted", err) + } + + pinned, places, err := pinImage(manifest, image) + if err != nil { + return out, err + } + say(fmt.Sprintf(" pinned %s, in %d place(s)", image, places)) + + container, _, err := containerIn(pinned, controlPlaneResourceIn(pinned)) + if err != nil { + return out, err + } + out.Container = container + if container == "" { + return out, fmt.Errorf( + "the %s module's container has no name, so nothing can be verified afterwards", + ControlPlaneModule) + } + + installed, err := registerAndAssign(ctx, o, control, ControlPlaneModule, pinned, say) + out.Installed = installed + if err != nil { + return out, err + } + + // The connections, before the push that would otherwise deliver random bytes for them. + delivered, err := deliverStores(ctx, o, control, pinned, substrate, say) + out.Delivered = delivered + if err != nil { + return out, err + } + + if out.Pushed, err = pushNode(ctx, o, control, say); err != nil { + return out, err + } + + if err := waitForContainer(ctx, control.run, o.Timeout, o.Wait, container, say); err != nil { + return out, err + } + // And it answers, which is the same question step 5 asked of the temporary one and for the + // same reason: `status` opens all three stores, so a reply proves the sealed connections it + // was given are the ones the substrate made. Asked of the NEW container — this is the only + // moment in the program where two control planes are running, and asking the wrong one would + // report the temporary one's health as the permanent one's. + answered, err := waitForTheControlPlane(ctx, control.run, o.Timeout, o.Wait, container, say) + if err != nil { + return out, err + } + out.Answered = answered + return out, nil +} + +// pinImage replaces the catalogue's placeholder digest with what the registry assigned. +// +// **Textual, and every place it appears.** A manifest may name its image in more than one resource +// — the catalogue's converted modules routinely carry a runtime container beside the application's +// — and the same reasoning as the bundle rewrite applies: replacing one and not the others leaves +// something pointing at an image nothing serves, and it fails half way through an apply rather +// than here. +// +// It refuses a manifest with no placeholder in it. That is not pedantry: a manifest already +// carrying a real digest is one somebody pinned by hand, and quietly registering it would install a +// control plane that is not the image this machine just published — which is the one thing this +// step exists to guarantee. +func pinImage(manifest []byte, reference string) ([]byte, int, error) { + places := bytes.Count(manifest, []byte(placeholderDigest)) + if places == 0 { + return nil, 0, fmt.Errorf( + "the %s module's manifest carries no placeholder digest (%s), so there is nothing to "+ + "pin to the image this machine just published.\n"+ + "A manifest already naming a digest was pinned by somebody else, to some other "+ + "build. Registering it would install a control plane that is not the one this "+ + "installer carried and pushed", ControlPlaneModule, placeholderDigest) + } + // The reference the registry gave back is `/@sha256:…`, and what the + // manifest holds is `@sha256:0…0`. Replacing only the digest would leave the + // manifest's own repository name in front of it — which may be `mesh-control` with no + // registry, and a runtime would then pull it from the internet. The whole reference moves. + var out bytes.Buffer + rest := manifest + for { + at := bytes.Index(rest, []byte(placeholderDigest)) + if at < 0 { + out.Write(rest) + break + } + // Back up over the repository this digest belongs to, which runs to the opening quote. + start := bytes.LastIndexByte(rest[:at], '"') + if start < 0 { + return nil, 0, fmt.Errorf( + "the %s module's manifest has a placeholder digest that is not inside a JSON "+ + "string, so the installer cannot tell what image it belongs to", ControlPlaneModule) + } + out.Write(rest[:start+1]) + out.WriteString(reference) + rest = rest[at+len(placeholderDigest):] + } + pinned := out.Bytes() + + // Read back. A substitution on text can catch more than it was aimed at, and the manifest is + // about to be handed to the mesh as the description of what it runs. + var checked map[string]any + if err := json.Unmarshal(pinned, &checked); err != nil { + return nil, 0, fmt.Errorf( + "pinning the %s module's image broke its manifest: %w", ControlPlaneModule, err) + } + if bytes.Contains(pinned, []byte(placeholderDigest)) { + return nil, 0, fmt.Errorf( + "the %s module's manifest still carries a placeholder digest after pinning", + ControlPlaneModule) + } + return pinned, places, nil +} + +// controlPlaneResourceIn is the id of the resource that runs the control plane. +// +// The manifest is written by the catalogue and the installer does not get to name its resources. +// What it can do is find the one container whose image is the one just pinned — and when a manifest +// declares exactly one container, that is the answer without any searching at all. +func controlPlaneResourceIn(manifest []byte) string { + var m struct { + Resources []struct { + ID string `json:"id"` + Type string `json:"type"` + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + return "" + } + var containers []string + for _, r := range m.Resources { + if r.Type == "container" { + containers = append(containers, r.ID) + } + } + if len(containers) == 1 { + return containers[0] + } + // More than one, so the name has to be guessed at rather than derived — and the catalogue's + // own convention for the resource that IS the module is `container`, with anything else beside + // it named for what it does. + for _, id := range containers { + if id == "container" || id == ControlPlaneModule || id == "control-plane" { + return id + } + } + return "" +} + +// deliverStores carries the substrate's own database connections into the module. +// +// The pairing is read from the manifest rather than assumed, so that whatever the catalogue calls +// these secrets is what is delivered. The installer does not guess that the secret holding the +// inventory connection is called `inventory`; it follows the manifest from the variable to the +// secret, and a manifest whose two ends do not meet is refused rather than half-delivered. +// +// **What is delivered is what the substrate already has, and only that.** The mesh generates an +// own-secret nobody supplied, which is right for something coming into existence and wrong for +// something that already exists. So every variable the module fills from a secret is looked up in +// the substrate's control plane: what it names is accepted, what it does not is left for the mesh +// to make. A store connection missing from the substrate is the one exception and is an error — +// a control plane that cannot open a context is not a control plane. +func deliverStores(ctx context.Context, o Options, control controlPlane, manifest []byte, + substrate *declaration.Declaration, say func(string)) ([]string, error) { + + wanted, err := secretsByVariableIn(manifest) + if err != nil { + return nil, err + } + if !anyStoreIn(wanted) { + return nil, fmt.Errorf( + "the %s module's manifest fills no %s… variable from a secret. A control plane reaches "+ + "each context through its own credential (novox/hq ADR 0008), so a manifest naming "+ + "none describes a control plane that can open nothing.\n"+ + "The shape this installer delivers into is a file per context, named by an "+ + "own-secret, with %s%s pointing at it — directly, or at where that file is "+ + "mounted inside the container", + ControlPlaneModule, storeVariablePrefix, storeVariablePrefix, storeFileSuffix) + } + + temporary, err := controlPlaneIn(substrate) + if err != nil { + return nil, err + } + + var delivered []string + for _, variable := range sortedKeys(wanted) { + secret := wanted[variable] + value := strings.TrimSpace(temporary.Env[variable]) + if value == "" { + if strings.HasPrefix(variable, storeVariablePrefix) { + return delivered, fmt.Errorf( + "the %s module wants %s and the bundle this installer produced does not name "+ + "one.\n"+ + "That connection is the substrate's, created at genesis — the mesh cannot "+ + "invent it and the installer will not guess at one", + ControlPlaneModule, variable) + } + // Not something the substrate made. The mesh generates its own, which is exactly what + // an own-secret is for; said so that nothing about the delivery is silent. + say(" the mesh will make " + secret + " — the substrate names no " + variable) + continue + } + + // Into the container as a file, because `secret accept` reads a file or a prompt and the + // installer has neither a terminal to be prompted at nor a way to write to a command's + // standard input through the runner every applier in this repository shares. + at := "/accepting-" + secret + if err := control.carrying(ctx, "mesh-accepting-"+secret, []byte(value), at); err != nil { + return delivered, err + } + if _, err := control.tell(ctx, "secret", "accept", o.Node, ControlPlaneModule, secret, + "--from", at); err != nil { + return delivered, err + } + delivered = append(delivered, secret) + say(" accepted " + secret + " — " + variable + ", as the substrate made it") + } + return delivered, nil +} + +// secretsByVariableIn maps each environment variable the module fills from a secret to that +// secret's name. +// +// Two shapes, because the catalogue uses both: +// +// - `MESH_STORE__FILE` in the container's environment, naming a path the process reads. +// The path may be the own-secret's own path, or — more usually — where that file is mounted +// inside the container, in which case the volumes say which is which. Following the mount is +// not a nicety: a manifest that keeps its secrets under `/var/lib/mesh/…` and mounts them at +// `/run/secrets/…` is the ordinary case, and matching on the path alone would find nothing and +// refuse a correct manifest. +// - `VAR=${secret:name}` inside a file resource the container reads its environment from, which +// is how a value that is not a path gets in at all. +// +// A `…_FILE` variable whose file nothing writes is refused: the mesh would seal nothing there and +// the process would find an empty file where a credential has to be, which presents as a container +// that will not start, a long way from the cause. +func secretsByVariableIn(manifest []byte) (map[string]string, error) { + var m struct { + OwnSecrets map[string]string `json:"own-secrets"` + Resources []struct { + Type string `json:"type"` + Path string `json:"path"` + Content string `json:"content"` + Env map[string]string `json:"env"` + Volumes []string `json:"volumes"` + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + return nil, fmt.Errorf("the %s module's manifest is not readable: %w", ControlPlaneModule, err) + } + + secretAt := map[string]string{} + for name, path := range m.OwnSecrets { + secretAt[path] = name + } + + wanted := map[string]string{} + for _, r := range m.Resources { + switch r.Type { + case "file": + for variable, secret := range secretsInContent(r.Content) { + wanted[variable] = secret + } + case "container": + inside := mountedFrom(r.Volumes) + for key, path := range r.Env { + if !strings.HasPrefix(key, storeVariablePrefix) || + !strings.HasSuffix(key, storeFileSuffix) { + continue + } + on := path + if from, mounted := inside[path]; mounted { + on = from + } + secret, named := secretAt[on] + if !named { + return nil, fmt.Errorf( + "the %s module's container reads %s from %s, and no own-secret of that "+ + "module writes that file.\n"+ + "So the mesh would seal nothing there and the control plane would find "+ + "an empty file where a connection string has to be. The manifest has to "+ + "name the two ends the same, directly or through a mount", + ControlPlaneModule, key, path) + } + wanted[strings.TrimSuffix(key, storeFileSuffix)] = secret + } + } + } + return wanted, nil +} + +// mountedFrom is where each path inside a container comes from outside it. +func mountedFrom(volumes []string) map[string]string { + inside := map[string]string{} + for _, volume := range volumes { + parts := strings.Split(volume, ":") + if len(parts) < 2 { + continue + } + inside[parts[1]] = parts[0] + } + return inside +} + +// secretsInContent finds `VAR=${secret:name}` lines in a file the container reads its environment +// from. +func secretsInContent(content string) map[string]string { + found := map[string]string{} + for _, line := range strings.Split(content, "\n") { + variable, value, is := strings.Cut(strings.TrimSpace(line), "=") + if !is { + continue + } + const opens = "${secret:" + if !strings.HasPrefix(value, opens) || !strings.HasSuffix(value, "}") { + continue + } + found[variable] = strings.TrimSuffix(strings.TrimPrefix(value, opens), "}") + } + return found +} + +// anyStoreIn reports whether any of these variables is a context's connection. +func anyStoreIn(wanted map[string]string) bool { + for variable := range wanted { + if strings.HasPrefix(variable, storeVariablePrefix) { + return true + } + } + return false +} + +func sortedKeys(m map[string]string) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/internal/bootstrap/control_test.go b/internal/bootstrap/control_test.go new file mode 100644 index 0000000..9a6a5e1 --- /dev/null +++ b/internal/bootstrap/control_test.go @@ -0,0 +1,275 @@ +package bootstrap + +import ( + "context" + "strings" + "testing" + "time" +) + +// Step 9 is where the control plane stops being a special case. These tests defend the two things +// that could go wrong quietly: pinning it to the wrong image, and delivering it store connections +// the mesh invented rather than the ones the substrate actually made. + +// theControlPlaneModule is the catalogue's manifest, trimmed to what this installer reads. +// +// A fixture rather than the file itself, unlike the substrate example the rewrite tests use: the +// catalogue is a different repository on a different branch, and a test that read it would pass or +// fail according to what somebody else had checked out. What it must stay faithful to is the +// SHAPE — the placeholder digest, the own-secret per context, the mount from the machine's path to +// the container's, and the environment file that fills what is not a path. +const theControlPlaneModule = `{ + "module": "mesh-control", + "version": "1", + "slug": "control", + "capabilities": ["container-runtime"], + "claims": [{"name": "the-control-plane", "scope": "mesh"}], + "own-secrets": { + "inventory": "/var/lib/mesh/mesh-control/inventory", + "identity": "/var/lib/mesh/mesh-control/identity", + "licences": "/var/lib/mesh/mesh-control/licences", + "broker": "/var/lib/mesh/mesh-control/broker", + "broker-management": "/var/lib/mesh/mesh-control/broker-management" + }, + "resources": [ + {"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-control", "mode": "0700"}, + {"id": "broker-env", "type": "file", "path": "/var/lib/mesh/mesh-control/broker.env", + "mode": "0600", + "content": "MESH_BROKER_AMQP=${secret:broker}\nMESH_BROKER_MANAGEMENT=${secret:broker-management}\nMESH_BROKER_ADDRESS=${machine:at}:5671\n"}, + {"id": "server", "type": "container", "name": "mesh-control", + "image": "mesh-control@` + placeholderDigest + `", + "network": "host", "args": ["serve"], + "env-file": ["/var/lib/mesh/mesh-control/broker.env"], + "env": { + "MESH_STORE_INVENTORY_FILE": "/run/secrets/inventory", + "MESH_STORE_IDENTITY_FILE": "/run/secrets/identity", + "MESH_STORE_LICENCES_FILE": "/run/secrets/licences", + "MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt" + }, + "volumes": [ + "mesh-broker-tls:/broker-tls:ro", + "/var/lib/mesh/mesh-control/inventory:/run/secrets/inventory:ro", + "/var/lib/mesh/mesh-control/identity:/run/secrets/identity:ro", + "/var/lib/mesh/mesh-control/licences:/run/secrets/licences:ro" + ]} + ] +}` + +const pushedReference = "127.0.0.1:5000/mesh-control@sha256:" + + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + +// **The whole reference moves, not only the digest.** The manifest's placeholder names a +// repository too, and replacing sixty-four zeros inside it would leave `mesh-control@sha256:…` +// with no registry in front — which a runtime would go to the internet for, and this mesh's +// control plane exists in no public registry by design. +func TestTheControlPlaneIsPinnedToWhatThisMeshsRegistryAssigned(t *testing.T) { + pinned, places, err := pinImage([]byte(theControlPlaneModule), pushedReference) + if err != nil { + t.Fatal(err) + } + if places != 1 { + t.Errorf("the placeholder was found in %d place(s)", places) + } + if !strings.Contains(string(pinned), `"image": "`+pushedReference+`"`) { + t.Errorf("the manifest does not name the pushed image:\n%s", pinned) + } + if strings.Contains(string(pinned), `"mesh-control@sha256:`) { + t.Errorf("the digest was replaced and the manifest's own repository name was left in "+ + "front of it, so nothing says which registry serves it:\n%s", pinned) + } +} + +// A manifest already naming a real digest was pinned by somebody else, to some other build. +// Registering it would install a control plane that is not the image this machine just published, +// which is the one thing this step exists to guarantee. +func TestAManifestAlreadyPinnedByHandIsRefused(t *testing.T) { + already := strings.Replace(theControlPlaneModule, placeholderDigest, + "sha256:"+strings.Repeat("9", 64), 1) + if _, _, err := pinImage([]byte(already), pushedReference); err == nil { + t.Fatal("a manifest already pinned to some other image was accepted") + } +} + +// Every placeholder moves. A manifest naming its image in a second resource — a runtime container +// beside the application's, which the catalogue's converted modules routinely carry — would +// otherwise be left half pinned, and fail inside an apply rather than here. +func TestEveryPlaceTheManifestNamesTheImageIsPinned(t *testing.T) { + twice := strings.Replace(theControlPlaneModule, + `{"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-control", "mode": "0700"},`, + `{"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-control", "mode": "0700"}, + {"id": "migrate", "type": "container", "name": "mesh-control-migrate", "run-once": true, + "image": "mesh-control@`+placeholderDigest+`", "args": ["migrate"]},`, 1) + + pinned, places, err := pinImage([]byte(twice), pushedReference) + if err != nil { + t.Fatal(err) + } + if places != 2 { + t.Errorf("the placeholder was found in %d place(s), and the manifest names it twice", places) + } + if strings.Contains(string(pinned), placeholderDigest) { + t.Error("a placeholder survived the pinning") + } +} + +// **The connections are the substrate's, and they are read out of the bundle that made them.** +// The mesh cannot invent them: they are the credentials the substrate created the databases with, +// and thirty-two random bytes in their place would leave the control plane unable to open a single +// context. The pairing is read from the manifest so that whatever the catalogue calls these +// secrets is what is delivered. +func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) { + wanted, err := secretsByVariableIn([]byte(theControlPlaneModule)) + if err != nil { + t.Fatal(err) + } + // **Through the mount.** The manifest keeps its secrets under /var/lib and the container reads + // them at /run/secrets. Matching on the path alone would find nothing and refuse a correct + // manifest, which is exactly the ordinary case in the catalogue. + for variable, secret := range map[string]string{ + "MESH_STORE_INVENTORY": "inventory", + "MESH_STORE_IDENTITY": "identity", + "MESH_STORE_LICENCES": "licences", + "MESH_BROKER_AMQP": "broker", + "MESH_BROKER_MANAGEMENT": "broker-management", + } { + if wanted[variable] != secret { + t.Errorf("%s would be accepted as %q, want %q", variable, wanted[variable], secret) + } + } + // And what the manifest fills from the machine rather than from a secret is left alone. + if _, claimed := wanted["MESH_BROKER_ADDRESS"]; claimed { + t.Error("the address the mesh composes from the machine was treated as a secret") + } + + // The values are the substrate's own, taken from the produced bundle rather than composed. + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + runtime := &asked{answer: aMeshThatAgrees(nil)} + control := controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second} + + delivered, err := deliverStores(context.Background(), Options{Node: "anchor"}, control, + []byte(theControlPlaneModule), rewritten.Declaration, func(string) {}) + if err != nil { + t.Fatal(err) + } + // Three stores and both halves of the broker: everything the substrate made and nothing else. + if len(delivered) != 5 { + t.Fatalf("%d values were delivered, and the substrate names five: %v", + len(delivered), delivered) + } + for _, secret := range delivered { + if !runtime.ran("secret accept anchor mesh-control " + secret + " --from") { + t.Errorf("%s was not accepted through `secret accept`: %v", secret, runtime.commands) + } + } +} + +// A secret the substrate did not make is left for the mesh to make, and said so. Every other +// secret in a mesh is one the mesh made; `secret accept` is only for what predates the mesh. +func TestASecretTheSubstrateNeverMadeIsLeftToTheMesh(t *testing.T) { + extra := strings.Replace(theControlPlaneModule, + `"broker": "/var/lib/mesh/mesh-control/broker",`, + `"broker": "/var/lib/mesh/mesh-control/broker", + "something-new": "/var/lib/mesh/mesh-control/something-new",`, 1) + extra = strings.Replace(extra, + `"content": "MESH_BROKER_AMQP=${secret:broker}\n`, + `"content": "MESH_SOMETHING_NEW=${secret:something-new}\nMESH_BROKER_AMQP=${secret:broker}\n`, 1) + + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + runtime := &asked{answer: aMeshThatAgrees(nil)} + control := controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second} + + var said []string + delivered, err := deliverStores(context.Background(), Options{Node: "anchor"}, control, + []byte(extra), rewritten.Declaration, func(line string) { said = append(said, line) }) + if err != nil { + t.Fatal(err) + } + for _, secret := range delivered { + if secret == "something-new" { + t.Error("a value the substrate never made was accepted as though it had") + } + } + if !strings.Contains(strings.Join(said, "\n"), "the mesh will make something-new") { + t.Errorf("nothing was said about the secret the mesh has to make: %v", said) + } +} + +// A manifest whose container reads a file no own-secret writes is refused. The mesh would seal +// nothing there and the control plane would find an empty file where a connection string has to +// be — which presents as a control plane that will not start, three steps from the cause. +func TestAConnectionFileNothingWritesIsRefused(t *testing.T) { + mismatched := strings.Replace(theControlPlaneModule, + `"inventory": "/var/lib/mesh/mesh-control/inventory",`, + `"inventory": "/var/lib/mesh/mesh-control/somewhere-else",`, 1) + + _, err := secretsByVariableIn([]byte(mismatched)) + if err == nil { + t.Fatal("a manifest whose two ends do not meet was accepted") + } + if !strings.Contains(err.Error(), "own-secret") { + t.Errorf("the refusal does not say which half is missing: %v", err) + } +} + +// A manifest asking for no store connections at all describes a control plane that can open +// nothing, and the refusal says what shape the installer delivers into — because the manifest is +// written in another repository and this is where the two have to agree. +func TestAManifestWantingNoStoresIsRefusedWithTheShapeItShouldHave(t *testing.T) { + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + runtime := &asked{answer: aMeshThatAgrees(nil)} + control := controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second} + + bare := `{"module":"mesh-control","version":"1","resources":[ + {"id":"container","type":"container","name":"mesh-control", + "image":"mesh-control@` + placeholderDigest + `"}]}` + + _, err = deliverStores(context.Background(), Options{Node: "anchor"}, control, + []byte(bare), rewritten.Declaration, func(string) {}) + if err == nil { + t.Fatal("a control plane that can open nothing was accepted") + } + if !strings.Contains(err.Error(), storeVariablePrefix+""+storeFileSuffix) { + t.Errorf("the refusal does not say what shape is expected: %v", err) + } +} + +// The permanent control plane is asked a question, not merely looked at — the same question the +// temporary one was asked at step 5, and for the same reason: `status` opens all three stores, so +// a reply proves the sealed connections it was given are the ones the substrate made. +func TestThePermanentControlPlaneIsAskedTheSameQuestion(t *testing.T) { + runtime := &asked{answer: aMeshThatAgrees(map[string]string{ + "module list": "", + "exec mesh-control /mesh-control": "1 node, 0 waiting\n", + })} + control := controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second} + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + + out, err := InstallControlPlane(context.Background(), + installing(t, catalogueWith(t, ControlPlaneModule, theControlPlaneModule)), + Deps{Run: runtime.run}, control, rewritten.Declaration, pushedReference, func(string) {}) + if err != nil { + t.Fatal(err) + } + if out.Answered != "1 node, 0 waiting" { + t.Errorf("the permanent control plane's reply is reported as %q", out.Answered) + } + if !runtime.ran("docker exec mesh-control " + controlPlaneBinary + " status") { + t.Errorf("the permanent control plane was never asked anything: %v", runtime.commands) + } + // And the module was registered with the digest, not with the placeholder. + if !runtime.ran("module add /mesh-control-module.json") { + t.Errorf("the module was never registered: %v", runtime.commands) + } +} diff --git a/internal/bootstrap/enrol.go b/internal/bootstrap/enrol.go new file mode 100644 index 0000000..facbda5 --- /dev/null +++ b/internal/bootstrap/enrol.go @@ -0,0 +1,316 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/system" +) + +// hereIs what `node list` says about a machine the mesh has heard from recently. +// +// mesh-control prints one of three words per node: "here", "never spoken", or "out of touch ". +// The installer waits for the first, and it is the only honest proof that the host agent is +// running: an enrolled machine whose host is not running looks exactly like an enrolled machine +// whose host has crashed, and both look exactly like a successful install until the first push +// silently applies nothing. +const hereIs = "here" + +// Enrolled is what step 6 did. +type Enrolled struct { + // Node is the name this machine is known by. + Node string + // Added is true when the mesh had no record and one was created. + Added bool + // Joined is true when this machine enrolled during this run. False on a re-run. + Joined bool + // Agent says how the host came to be running: what was found, or what was started. + Agent string +} + +// Enrol makes this machine the mesh's first node, and gets the host agent running on it. +// +// **The mesh is running and nothing has joined it.** Steps 1 to 5 leave a store, a broker and a +// control plane that answers — a mesh of one node in the sense that it has one machine and zero +// node records. Everything after this point is the control plane being *told* things, and none of +// it reaches a machine until a host is running there to hear it. +// +// Four things, in this order, each asked before it is done: +// +// 1. a node record, unless `node list` already shows one +// 2. a one-time token, unless this machine already holds an identity +// 3. `mesh-host enrol`, which is the machine presenting the identity it already had +// 4. the host agent running, and the mesh saying it has heard from it +// +// **Step 3 is the one that cannot be undone by re-running.** A machine that has enrolled holds an +// identity the mesh has recorded, and enrolling again would replace it with a second one the mesh +// does not know — `mesh-host enrol` refuses exactly this, and so does the check here, one layer +// earlier and with a sentence about what to do. +// +// `control.run` is this machine's runner and not only the control plane's: the same injected +// Runner reaches the container, the service manager and the host binary, which is what lets the +// whole of this be tested without any of the three. +func Enrol(ctx context.Context, o Options, sys system.System, control controlPlane, + say func(string)) (Enrolled, error) { + + out := Enrolled{Node: o.Node} + if strings.TrimSpace(o.Node) == "" { + return out, errors.New( + "this machine has no name to be known by. Give one with --node; it is what the mesh " + + "records, what a token is issued against, and what every later assignment names") + } + + // 1. The record. + nodes, err := control.tell(ctx, "node", "list") + if err != nil { + return out, err + } + if mentions(nodes, o.Node) { + say(" already a node " + o.Node) + } else { + if _, err := control.tell(ctx, "node", "add", o.Node); err != nil { + return out, err + } + out.Added = true + say(" node record " + o.Node) + } + + // 2 and 3. The identity, and being known. + // + // Asked of this machine's own identity file rather than of the mesh, because the two answer + // different questions: the mesh knows whether a record exists, and only the machine knows + // whether it holds the key that record names. + where := identity.Path(o.State) + switch mine, err := identity.Load(where); { + case err == nil && mine.Node == o.Node: + say(" already enrolled " + o.Node + " — " + where) + case err == nil: + return out, fmt.Errorf( + "this machine is already node %q and was asked to become %q.\n"+ + "Re-enrolling replaces the identity the mesh has recorded, so it is not something "+ + "an installer does on its own. Run with --node %s, or remove %s and start over "+ + "deliberately", mine.Node, o.Node, mine.Node, where) + case !errors.Is(err, identity.ErrNoIdentity): + return out, fmt.Errorf("cannot read this machine's identity at %s: %w", where, err) + default: + said, err := control.tell(ctx, "token", "issue", "--node", o.Node) + if err != nil { + return out, err + } + token, err := tokenIn(said) + if err != nil { + return out, err + } + joining, cancel := context.WithTimeout(ctx, o.Wait) + joined, err := control.run(joining, o.Host, "enrol", "--token", token, "--state", o.State) + cancel() + if err != nil { + return out, fmt.Errorf( + "%s would not enrol this machine: %w\n%s\n"+ + "The token is one-time and has now been spent; running this again issues "+ + "another, so a re-run is safe", o.Host, err, indent(strings.TrimSpace(joined))) + } + if !strings.Contains(joined, "enrolled as "+o.Node) { + // Exit zero and no such sentence. Refused rather than believed: the host says exactly + // this line on success, and something that succeeded without saying it did something + // else. + return out, fmt.Errorf( + "%s exited happily and did not say it enrolled as %s:\n%s", + o.Host, o.Node, indent(strings.TrimSpace(joined))) + } + out.Joined = true + say(" enrolled as " + o.Node) + } + + // 4. The agent. + agent, err := runTheHost(ctx, o, sys, control, say) + if err != nil { + return out, err + } + out.Agent = agent + return out, nil +} + +// runTheHost makes sure something on this machine is listening to the mesh, and proves it. +// +// **The installer does not install the service, and says so.** A unit file is a packaging decision +// — where the binary lives, which user it runs as, what it is called — and an installer that +// invented one would be putting a file on the machine that whatever installed `mesh-host` will +// later disagree with. So this starts a unit that is already there and refuses when there is none. +// +// It goes through the host's OWN system abstraction rather than running systemctl itself, for the +// reason `ApplyBundle` gives about applying: two implementations of "is this service running" is +// how the installer and the host come to disagree about a machine. It also gets the right refusal +// for free — `ServiceState` treats a unit that does not exist as an error and never as "stopped", +// which is exactly the distinction that matters here. +// +// --host-in-background is the lab's arrangement, kept because the lab is what exercises this and +// it has no service. It is loud about what it is, because a host started this way is gone at the +// next reboot and the mesh would go quiet for reasons nobody would connect to an install. +func runTheHost(ctx context.Context, o Options, sys system.System, control controlPlane, + say func(string)) (string, error) { + + // **Asked of both, because either one alone lies.** + // + // A process in the table may be wedged and never collect anything, which is why this asked the + // mesh instead. But the mesh having heard from a node proves only that something spoke to it + // *once*, and `mesh-host enrol` — run moments earlier, by this very step — is itself that + // something. So straight after enrolling, the mesh has always heard from this machine and no + // agent is running; skipping the start on that signal alone is exactly wrong. + // + // What it costs is silent and total. Every later step is the control plane being *told* + // things, and nothing it is told reaches a machine with no agent to collect it: the push at + // step 7 is accepted, the mesh records the module, and no container is ever created. It + // surfaces three minutes later as "the registry is not there at all" — one step from its + // cause, looking nothing like it. + if heard, err := heardFrom(ctx, control, o.Node); err != nil { + return "", err + } else if heard { + running, err := agentIsRunning(ctx, o, sys, control) + if err != nil { + return "", err + } + if running { + say(" host running the mesh has heard from " + o.Node) + return "already running", nil + } + say(" host running the mesh has heard from " + o.Node + ", and no agent is running " + + "here — enrolling speaks once, which is not the same thing") + } + + how := "" + switch { + case o.HostInBackground: + // Detached, and not waited for: this process runs until the machine stops, so a runner + // that captures output would never return. `nohup … &` through a shell is how the lab + // starts it and is deliberately the same command. + started, cancel := context.WithTimeout(ctx, o.Timeout) + _, err := control.run(started, "sh", "-c", + fmt.Sprintf("nohup %s run --state %s >> /var/log/mesh-host.log 2>&1 &", o.Host, o.State)) + cancel() + if err != nil { + return "", fmt.Errorf("cannot start %s in the background: %w", o.Host, err) + } + how = "started in the background" + say(" host running started in the background — THIS DOES NOT SURVIVE A REBOOT.") + say(" A real machine needs " + o.HostService + " installed and enabled.") + + default: + state, err := sys.ServiceState(ctx, control.run, o.HostService) + if err != nil { + return "", fmt.Errorf( + "the mesh has not heard from %s and this machine has no %s to start: %w\n"+ + "The host has to be running for anything the mesh says to reach this machine. "+ + "Install the service that supervises it and run this again — every step "+ + "before this one will say it is already done. In a lab, --host-in-background "+ + "starts it unsupervised instead, and that is not an install", + o.Node, o.HostService, err) + } + if state != "running" { + if err := sys.SetServiceState(ctx, control.run, o.HostService, "running"); err != nil { + return "", fmt.Errorf("cannot start %s: %w", o.HostService, err) + } + how = "started " + o.HostService + say(" host running started " + o.HostService) + } else { + // Running, and the mesh has not heard from it. Not an error yet — it may have started + // a second ago — so it is waited for below like everything else that is merely + // starting. + how = o.HostService + " was already running" + say(" host running " + o.HostService + " is running") + } + // At boot as well, or the machine comes back without a mesh and nothing says why. + if boot, err := sys.ServiceBoot(ctx, control.run, o.HostService); err == nil && boot != "enabled" { + if err := sys.SetServiceBoot(ctx, control.run, o.HostService, "enabled"); err != nil { + return "", fmt.Errorf("cannot make %s start at boot: %w", o.HostService, err) + } + say(" host at boot " + o.HostService + " enabled") + } + } + + // Read back (novox/hq ADR 0018). A service that started and a mesh that has heard from a node + // are two different facts, and only the second is the one every later step rests on. + deadline := time.Now().Add(o.Wait) + for { + heard, err := heardFrom(ctx, control, o.Node) + if err != nil { + return "", err + } + if heard { + say(" the mesh hears " + o.Node) + return how, nil + } + if time.Now().After(deadline) { + return "", fmt.Errorf( + "%s is running and the mesh has not heard from %s after %s.\n"+ + "The host links to the broker at the address the token carried — if that "+ + "address is not one this machine can reach, this is where it shows. Read the "+ + "host's own output, and check MESH_BROKER_ADDRESS in the bundle this "+ + "installer wrote", o.HostService, o.Node, o.Wait) + } + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(answerEvery): + } + } +} + +// agentIsRunning is whether a host agent is on this machine now — the other half of the question +// the mesh cannot answer. +// +// Deliberately not an error when there is nothing to find: "no unit here" and "not running" are +// both simply *not running* to this caller, and the branch that starts one says far more about a +// missing unit than this could, naming what to install. +func agentIsRunning(ctx context.Context, o Options, sys system.System, control controlPlane) (bool, error) { + if o.HostInBackground { + // No unit to ask, so the process table is all there is. The exit status is the whole + // answer; anything it printed is not this function's business. + if _, err := control.run(ctx, "pgrep", "-f", o.Host+" run"); err != nil { + return false, nil + } + return true, nil + } + state, err := sys.ServiceState(ctx, control.run, o.HostService) + if err != nil { + return false, nil + } + return state == "running", nil +} + +// heardFrom asks the mesh whether this node has spoken to it. +func heardFrom(ctx context.Context, control controlPlane, node string) (bool, error) { + listing, err := control.tell(ctx, "node", "list") + if err != nil { + return false, err + } + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == node { + return fields[1] == hereIs, nil + } + } + return false, nil +} + +// tokenIn finds the token in what `token issue` said. +// +// The same rule the lab uses: the one long unbroken line. `token issue` prints an explanation +// around it, and a token is a signed blob with no spaces in it, so "long and unbroken" identifies +// it without this having to know the format — which is what keeps the installer from having an +// opinion about a thing the control plane owns. +func tokenIn(said string) (string, error) { + for _, line := range strings.Split(said, "\n") { + line = strings.TrimSpace(line) + if len(line) > 100 && !strings.ContainsAny(line, " \t") { + return line, nil + } + } + return "", fmt.Errorf( + "the mesh issued a token and there is no token in what it said:\n%s", + indent(strings.TrimSpace(said))) +} diff --git a/internal/bootstrap/enrol_test.go b/internal/bootstrap/enrol_test.go new file mode 100644 index 0000000..cad2fa9 --- /dev/null +++ b/internal/bootstrap/enrol_test.go @@ -0,0 +1,263 @@ +package bootstrap + +import ( + "context" + "crypto/ed25519" + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/system" +) + +// Step 6 is where the mesh stops being something running on a machine and starts being something +// the machine belongs to. What these tests defend is that it cannot happen twice, and that +// "installed" is never claimed for a machine the mesh has not actually heard from. + +// alreadyEnrolled writes an identity file, as `mesh-host enrol` leaves behind. +func alreadyEnrolled(t *testing.T, node string) string { + t.Helper() + state := filepath.Join(t.TempDir(), "state.json") + mine, err := identity.Generate(node) + if err != nil { + t.Fatal(err) + } + // A whole membership, because an identity that cannot reach its mesh is refused on the way in + // — which is the right refusal and not the one being tested here. + signer, _, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + mine.Membership = identity.Membership{ + Broker: "192.0.2.10:5671", Fingerprint: "sha256:whatever", + Signer: signer, Password: "issued-at-enrolment", + } + if err := identity.Save(identity.Path(state), mine); err != nil { + t.Fatal(err) + } + return state +} + +func arch(t *testing.T) system.System { + t.Helper() + chosen, err := system.For("arch") + if err != nil { + t.Fatal(err) + } + return chosen +} + +// A machine that has already enrolled is not enrolled again, and no token is spent on it. The +// installer is run over and over; a second identity is one the mesh does not know, and the mesh +// believes the first. +func TestAMachineThatHasAlreadyEnrolledIsNotEnrolledAgain(t *testing.T) { + runtime := &asked{answer: func(name string, args []string) (string, error) { + joined := strings.Join(args, " ") + switch { + case strings.Contains(joined, "node list"): + return "anchor here 01J0\n", nil + // An agent is running here too. Both halves are needed: enrolling makes the mesh hear from + // a machine once, so the first answer alone also describes a machine with no agent at all. + case name == "pgrep": + return "4242\n", nil + } + return "", fmt.Errorf("unexpected: %s %v", name, args) + }} + + out, err := Enrol(context.Background(), Options{ + Node: "anchor", State: alreadyEnrolled(t, "anchor"), Timeout: time.Second, + Host: "/usr/local/bin/mesh-host", HostInBackground: true, + }, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err != nil { + t.Fatal(err) + } + if out.Joined || out.Added { + t.Error("a machine that had already enrolled enrolled again") + } + if runtime.ran("token issue") { + t.Errorf("a token was issued for a machine that already holds an identity: %v", + runtime.commands) + } + if out.Agent != "already running" { + t.Errorf("the host agent is reported as %q", out.Agent) + } +} + +// The mesh having heard from a machine is not the same as an agent running on it, and enrolling is +// itself the thing that makes the mesh hear. Taken as proof of life it ends the step believing an +// agent it never started, and everything after is the control plane being told things that never +// reach the machine: the next push is accepted, recorded, and applied by nobody. +func TestAMeshThatHasHeardFromAMachineWithNoAgentStartsOne(t *testing.T) { + runtime := &asked{answer: func(name string, args []string) (string, error) { + joined := strings.Join(args, " ") + switch { + case strings.Contains(joined, "node list"): + // Exactly what enrolling leaves behind, with nothing running. + return "anchor here 01J0\n", nil + case name == "pgrep": + return "", fmt.Errorf("exit status 1") + case name == "sh": + return "", nil + } + return "", fmt.Errorf("unexpected: %s %v", name, args) + }} + + out, err := Enrol(context.Background(), Options{ + Node: "anchor", State: alreadyEnrolled(t, "anchor"), Timeout: time.Second, + Host: "/usr/local/bin/mesh-host", HostInBackground: true, + }, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err != nil { + t.Fatal(err) + } + if out.Agent == "already running" { + t.Fatal("a machine with no agent was reported as already running it") + } + if !runtime.ran("nohup") { + t.Errorf("no agent was started on a machine that has none: %v", runtime.commands) + } +} + +// A machine already enrolled under ANOTHER name is refused, with what to do about it. Re-enrolling +// replaces the identity the mesh recorded, which is a deliberate act and not something an +// installer does on its own. +func TestAMachineEnrolledUnderAnotherNameIsRefused(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if strings.Contains(strings.Join(args, " "), "node list") { + return "somewhere-else here 01J0\n", nil + } + return "", nil + }} + + _, err := Enrol(context.Background(), Options{ + Node: "anchor", State: alreadyEnrolled(t, "somewhere-else"), Timeout: time.Second, + }, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err == nil { + t.Fatal("a machine already enrolled as something else was enrolled again") + } + for _, wanted := range []string{"somewhere-else", "--node"} { + if !strings.Contains(err.Error(), wanted) { + t.Errorf("the refusal does not mention %q:\n%v", wanted, err) + } + } +} + +// **The mesh having heard from the node is the proof, not a process existing.** A host that is +// running and cannot reach the broker looks exactly like a successful install until the first push +// silently applies nothing — which is the class of fault this whole program exists to stop being +// found late. +func TestAHostThatIsRunningAndUnheardOfIsNotAnInstall(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + runtime := &asked{answer: func(_ string, args []string) (string, error) { + joined := strings.Join(args, " ") + switch { + case strings.Contains(joined, "node list"): + // Enrolled, and never spoken. + return "anchor never spoken 01J0\n", nil + case args[0] == "show": + return "LoadState=loaded\nActiveState=active\n", nil + case args[0] == "is-enabled": + return "enabled\n", nil + } + return "", nil + }} + + _, err := Enrol(context.Background(), Options{ + Node: "anchor", State: alreadyEnrolled(t, "anchor"), HostService: "mesh-host.service", + Timeout: time.Second, Wait: 0, + }, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err == nil { + t.Fatal("a node the mesh has never heard from was reported enrolled and running") + } + if !strings.Contains(err.Error(), "MESH_BROKER_ADDRESS") { + t.Errorf("the failure does not name the thing that is silently fatal when wrong:\n%v", err) + } +} + +// A machine with no service to start is refused, and the refusal says what is missing rather than +// inventing a unit file. What a unit says is a packaging decision, and an installer writing one +// would put a file on the machine that whatever installed the host will disagree with. +func TestAMachineWithNoHostServiceIsRefusedRatherThanGivenOne(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + joined := strings.Join(args, " ") + switch { + case strings.Contains(joined, "node list"): + return "anchor never spoken 01J0\n", nil + case args[0] == "show": + // systemd knows nothing about it. + return "LoadState=not-found\nActiveState=inactive\n", nil + } + return "", nil + }} + + _, err := Enrol(context.Background(), Options{ + Node: "anchor", State: alreadyEnrolled(t, "anchor"), HostService: "mesh-host.service", + Timeout: time.Second, Wait: 0, + }, arch(t), controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err == nil { + t.Fatal("a machine with no host service was reported as having a running host") + } + for _, wanted := range []string{"mesh-host.service", "--host-in-background"} { + if !strings.Contains(err.Error(), wanted) { + t.Errorf("the refusal does not mention %q:\n%v", wanted, err) + } + } +} + +// A machine with no name is refused before anything is said to the mesh. The name is what the +// record, the token, the assignment and the push all name, and one the installer invented would +// match nothing anybody types anywhere else. +func TestAMachineWithNoNameIsRefusedBeforeAnythingIsAsked(t *testing.T) { + runtime := &asked{answer: func(string, []string) (string, error) { + return "", fmt.Errorf("nothing should have been asked") + }} + _, err := Enrol(context.Background(), Options{Timeout: time.Second}, arch(t), + controlPlane{container: "temp-mesh-control", run: runtime.run}, func(string) {}) + if err == nil { + t.Fatal("a machine with no name was enrolled") + } + if len(runtime.commands) != 0 { + t.Errorf("the mesh was asked something first: %v", runtime.commands) + } +} + +// The token is found in what the mesh said, by the rule the lab uses: the one long unbroken line. +// The installer does not parse a format the control plane owns. +func TestTheTokenIsFoundInWhatTheMeshSaid(t *testing.T) { + said := "a token for anchor, good once:\n\n " + strings.Repeat("t", 240) + "\n\n" + + "carry it to the machine and run: mesh-host enrol --token \n" + token, err := tokenIn(said) + if err != nil { + t.Fatal(err) + } + if token != strings.Repeat("t", 240) { + t.Errorf("the token was read as %q", token) + } + + if _, err := tokenIn("nothing here that looks like one\n"); err == nil { + t.Fatal("an answer with no token in it was accepted") + } +} + +// A listing's name is matched as a whole word at the start of a line, so `registry` is not found +// inside `registry-mirror`. A substring match would report a module installed that is not, and the +// installer would skip creating it. +func TestAListingIsMatchedByNameAndNotBySubstring(t *testing.T) { + listing := "registry-mirror 1 built abc\nother 1 built def\n" + if mentions(listing, "registry") { + t.Error("registry-mirror was read as registry") + } + if !mentions(listing, "registry-mirror") { + t.Error("registry-mirror was not found") + } +} diff --git a/internal/bootstrap/load.go b/internal/bootstrap/load.go new file mode 100644 index 0000000..4e1fc71 --- /dev/null +++ b/internal/bootstrap/load.go @@ -0,0 +1,199 @@ +package bootstrap + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/novox/mesh-host/internal/image" +) + +// Loaded is the control plane's image on this machine. +type Loaded struct { + // ID is what THIS RUNTIME holds the image as, read back from it after the load. It is what the + // bundle names, and outside a dry run it is never a prediction — see Load. + ID string + // Archive is what the carried tar calls the same image. Kept because the two differ in + // practice, and a report showing only one of them cannot say that they did. Never what the + // bundle names. + Archive string + // Tag is the name the runtime is asked by. Load-bearing rather than decoration: it is the one + // name that survives `docker save` and `docker load` unchanged. + Tag string + // Tags is everything the archive was called when it was saved. + Tags []string + // Held is true when the machine already held it and nothing moved. + Held bool + // Predicted is true only on a dry run, where nothing was loaded and ID is therefore the + // archive's id — which is not necessarily the one this machine would end up with. + Predicted bool +} + +// Load puts the carried control-plane image into this machine's container runtime, and reports +// what the runtime decided to call it. +// +// **The digest of a configuration is not portable across runtimes, and that is why the id is read +// back rather than predicted.** An image id is the sha256 of the image's configuration document, +// and a runtime REWRITES that document as it loads: a newer Docker saves in one format, an older +// one stores it in another, and the same layers come out under a different name. Measured on a +// live raise, an image saved as `sha256:b86bb81c…` on a workstation was loaded as +// `sha256:2dc21904…` on the machine it was carried to. +// +// This code used to read the id out of the tar before the runtime was asked anything and use it +// for both idempotence and the bundle. That is right on the machine the image was built on and +// wrong on every machine it is carried to — which is every machine this program exists for. The +// bundle would have named an image the machine does not hold; nothing serves an image named by +// the digest of its own configuration, which is the whole point of naming one that way; and the +// apply would have stopped inside a pull that cannot succeed. The lab hit exactly this. +// +// **So the image is identified by its TAG.** A tag is ordinary metadata the tar carries through +// unchanged, and asking the runtime what a tag resolves to is asking the only party entitled to +// answer. The tag never reaches the bundle — a pinned bundle may not rely on one +// (novox/hq ADR 0006) — it is how the id is obtained, not what is written down. +// +// **Idempotence is decided from what the runtime holds.** The tag is asked before the load and +// again after: the same id either side means nothing moved, which is a fact about this machine +// rather than a guess about the file. A tag that already resolves means the image is already +// held, and nothing is loaded at all. +// +// It reads back (novox/hq ADR 0018). A load that reported success and left nothing there is a +// failure, not a convergence. +func Load(ctx context.Context, run Runner, dryRun bool, say func(string)) (Loaded, error) { + saved, err := image.Saved() + if err != nil { + return Loaded{}, err + } + return loadImage(ctx, run, saved, dryRun, say) +} + +// loadImage is Load with the carried bytes handed in, so the whole path can be tested against a +// saved image a test builds rather than against whatever a particular build embedded. +func loadImage(ctx context.Context, run Runner, saved []byte, dryRun bool, say func(string)) (Loaded, error) { + archiveID, err := image.ArchiveID(saved) + if err != nil { + return Loaded{}, err + } + loaded := Loaded{Archive: archiveID, Tags: image.Tags(saved)} + + // **An untagged archive is a build-time fault, refused here rather than worked around.** + // Without a tag there is no portable name to ask the runtime about, and the only thing left is + // scraping the sentence `docker load` prints for a person — which differs between runtime + // versions and is exactly the kind of guess this whole step exists to stop making. The release + // target tags the image; an installer built without one was built wrong. + loaded.Tag = firstOr(loaded.Tags, "") + if loaded.Tag == "" { + return loaded, fmt.Errorf( + "the carried control-plane image has no tag, so there is no portable name to ask this "+ + "machine's runtime what id it gave it.\n"+ + "An image id is the digest of the image's configuration and a runtime rewrites "+ + "that as it loads, so the id in the archive (%s) is not necessarily the id this "+ + "machine will hold — and a bundle naming the wrong one names an image nothing "+ + "serves. Rebuild the installer with a tagged image: `make bootstrap "+ + "IMAGE=:`", archiveID) + } + + // Already held? Asked of the runtime, by the tag, before anything is written anywhere. + before, err := idOfImage(ctx, run, loaded.Tag) + if err != nil { + return loaded, err + } + if before != "" { + loaded.ID, loaded.Held = before, true + say(" already held " + before + " as " + loaded.Tag + " — nothing loaded") + sayIfDifferent(say, archiveID, before) + return loaded, nil + } + + if dryRun { + // Nothing is loaded, so the runtime has not been asked to decide anything — and what it + // would decide cannot be worked out from here. Said, rather than quietly guessed at. + loaded.ID, loaded.Predicted = archiveID, true + say(fmt.Sprintf(" would load %s (%d bytes) as %s", archiveID, len(saved), loaded.Tag)) + say(" NOT THE FINAL ID a runtime rewrites an image's configuration as it loads, and an") + say(" id is that configuration's digest. The bundle names what the") + say(" runtime answers for " + loaded.Tag + " afterwards, which a dry run") + say(" cannot ask for without loading.") + return loaded, nil + } + + // Through a file rather than through stdin: the runner this repository shares runs a command + // and captures its output, and giving it a second mouth for one caller would change every + // applier's contract for the sake of one step (internal/apply's Runner). + tarball, err := os.CreateTemp("", "mesh-control-*.tar") + if err != nil { + return loaded, fmt.Errorf("nowhere to put the carried image while loading it: %w", err) + } + defer os.Remove(tarball.Name()) + + if _, err := tarball.Write(saved); err != nil { + tarball.Close() + return loaded, fmt.Errorf("cannot write the carried image to %s: %w", tarball.Name(), err) + } + if err := tarball.Close(); err != nil { + return loaded, fmt.Errorf("cannot finish writing %s: %w", tarball.Name(), err) + } + + if _, err := run(ctx, "docker", "load", "--input", tarball.Name()); err != nil { + return loaded, fmt.Errorf( + "the container runtime would not load the carried control-plane image: %w", err) + } + + // Read back, and THIS is the answer the bundle is rewritten to. + after, err := idOfImage(ctx, run, loaded.Tag) + if err != nil { + return loaded, err + } + if after == "" { + return loaded, fmt.Errorf( + "the load reported success and this machine holds nothing called %s.\n"+ + "The bundle names the control plane by the id this runtime assigned, so there is "+ + "nothing to name. Check what `docker load` actually took", loaded.Tag) + } + loaded.ID = after + say(" loaded " + after + " as " + loaded.Tag) + sayIfDifferent(say, archiveID, after) + return loaded, nil +} + +// sayIfDifferent reports the archive's own id when the runtime chose another. +// +// Said every time it happens, because it is surprising, it is ordinary, and somebody comparing +// this report against `docker images` on the machine the image was built on would otherwise +// conclude that the wrong image had been carried. +func sayIfDifferent(say func(string), archiveID, held string) { + if archiveID == held { + return + } + say(" the archive says " + archiveID) + say(" this runtime stored the same image under a different configuration, " + + "which is ordinary — the bundle names what the machine holds") +} + +// idOfImage asks the runtime what it holds under a name, or empty if it holds nothing. +// +// It asks for the id back rather than reading the exit code, because the id is what is wanted and +// an exit code is not it. Absent is an answer and not a failure: every other reason the runtime +// might refuse looks the same from here, which is why preflight proves the runtime answers before +// this runs rather than this trying to tell the two apart from an exit status. +// +// What it will not do is accept an answer that is not an image id. That answer becomes the name +// the bundle applies on a machine with no mesh to check anything against, so it is checked here +// where the refusal can say whose mistake it is. +func idOfImage(ctx context.Context, run Runner, name string) (string, error) { + out, err := run(ctx, "docker", "image", "inspect", "--format", "{{.Id}}", name) + if err != nil { + return "", nil + } + got := strings.TrimSpace(firstLineOf(out)) + if got == "" { + return "", nil + } + if !isImageID(got) { + return "", fmt.Errorf( + "asked what this machine holds as %q, the runtime answered %q, which is not an image "+ + "id. The bundle would name the control plane by that answer, and it is refused "+ + "rather than written down", name, got) + } + return got, nil +} diff --git a/internal/bootstrap/load_test.go b/internal/bootstrap/load_test.go new file mode 100644 index 0000000..b37fdc2 --- /dev/null +++ b/internal/bootstrap/load_test.go @@ -0,0 +1,313 @@ +package bootstrap + +import ( + "archive/tar" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/image" +) + +// Docker is never required here. What is being tested is which commands the installer issues and +// what it concludes from the answers, so the runtime is injected the way `internal/apply` injects +// its Runner (novox/hq ADR 0017). Behaviour against a real runtime is proved in the lab. + +// asked records every command, so a test can assert that something was NOT run — which is the +// whole of what idempotence means here. +type asked struct { + commands []string + answer func(name string, args []string) (string, error) +} + +func (a *asked) run(_ context.Context, name string, args ...string) (string, error) { + a.commands = append(a.commands, strings.TrimSpace(name+" "+strings.Join(args, " "))) + if a.answer == nil { + return "", errors.New("this test did not expect any command to be run") + } + return a.answer(name, args) +} + +func (a *asked) ran(fragment string) bool { + for _, command := range a.commands { + if strings.Contains(command, fragment) { + return true + } + } + return false +} + +// savedImageFixture builds what `docker save` produces, tagged `mesh-control:test` unless a test +// asks for something else. Pass no tags for an archive saved without one. +func savedImageFixture(t *testing.T, digest string, tags ...string) []byte { + t.Helper() + if tags == nil { + tags = []string{"mesh-control:test"} + } + entries, err := json.Marshal([]struct { + Config string + RepoTags []string + }{{Config: digest + ".json", RepoTags: tags}}) + if err != nil { + t.Fatal(err) + } + + var buffer bytes.Buffer + writer := tar.NewWriter(&buffer) + if err := writer.WriteHeader(&tar.Header{ + Name: "manifest.json", Mode: 0o644, Size: int64(len(entries)), + }); err != nil { + t.Fatal(err) + } + if _, err := writer.Write(entries); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + return buffer.Bytes() +} + +// fixtureDigest is what the ARCHIVE calls the image, and runtimeDigest is what a runtime calls it +// after loading the same bytes. They differ on purpose, because they differ in reality: an image +// id is the digest of the image's configuration, and a runtime rewrites that configuration as it +// loads. Measured on a live raise, `mesh-control:development` was `sha256:b86bb81c…` on the +// workstation that saved it and `sha256:2dc21904…` on the machine that loaded it. +const ( + fixtureDigest = "3333333333333333333333333333333333333333333333333333333333333333" + runtimeDigest = "4444444444444444444444444444444444444444444444444444444444444444" +) + +// **The id the bundle is named by comes from the RUNTIME, not from the archive.** +// +// This is the test for the fault that took the lab down. The installer used to read the id out of +// the carried tar and use it for the bundle, which is correct on the machine the image was built +// on and wrong on every machine it is carried to — and a bundle naming an id the machine does not +// hold names an image nothing can serve, because an image named by the digest of its own +// configuration is by definition served by nobody. The apply then stops inside a pull that cannot +// succeed, three steps from the cause. +// +// So the image is identified by its TAG, which survives save and load unchanged, and the runtime +// is asked what that tag resolves to. +func TestTheIdComesFromTheRuntimeAndNotFromTheArchive(t *testing.T) { + runtime := &asked{} + inspected := 0 + runtime.answer = func(_ string, args []string) (string, error) { + switch { + case len(args) > 1 && args[0] == "image" && args[1] == "inspect": + inspected++ + if inspected == 1 { + // Nothing held yet. + return "", errors.New("Error: No such image") + } + // Loaded — and stored under a configuration of the runtime's own making. + return "sha256:" + runtimeDigest + "\n", nil + case len(args) > 0 && args[0] == "load": + return "Loaded image: mesh-control:test\n", nil + } + return "", fmt.Errorf("unexpected command: %v", args) + } + + var said []string + loaded, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), false, func(line string) { said = append(said, line) }) + if err != nil { + t.Fatal(err) + } + if loaded.ID != "sha256:"+runtimeDigest { + t.Errorf("the bundle would name %q; this machine holds sha256:%s", loaded.ID, runtimeDigest) + } + if loaded.Archive != "sha256:"+fixtureDigest { + t.Errorf("the archive's own id is reported as %q", loaded.Archive) + } + if loaded.Predicted { + t.Error("a real run reported its id as a prediction") + } + // The runtime was asked BY THE TAG, which is the only name that survives the transfer. + if !runtime.ran("docker image inspect --format {{.Id}} mesh-control:test") { + t.Errorf("the runtime was never asked what the tag resolves to: %v", runtime.commands) + } + // And the difference is said out loud, or somebody comparing this against `docker images` on + // the build machine concludes the wrong image was carried. + if !strings.Contains(strings.Join(said, "\n"), "the archive says") { + t.Errorf("nothing was said about the two ids differing: %v", said) + } +} + +// A machine that already holds the image is not loaded again, and says so. +// +// This is the idempotence the installer's usefulness rests on: it is run over and over while +// somebody gets a machine working, and a step that did its work again every time would be +// indistinguishable from one that had never run. **Decided from what the runtime holds under the +// tag, not from what the archive predicts** — a predicted id cannot answer this question at all on +// a machine whose runtime rewrites configurations. +func TestAnImageThisMachineAlreadyHoldsIsNotLoadedAgain(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if len(args) > 1 && args[0] == "image" && args[1] == "inspect" { + return "sha256:" + runtimeDigest + "\n", nil + } + return "", fmt.Errorf("unexpected command: %v", args) + }} + + var said []string + loaded, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), false, func(line string) { said = append(said, line) }) + if err != nil { + t.Fatal(err) + } + + if !loaded.Held { + t.Error("the machine already held the image and the load did not say so") + } + if loaded.ID != "sha256:"+runtimeDigest { + t.Errorf("the id reported for an already-held image is %q, and the machine holds sha256:%s", + loaded.ID, runtimeDigest) + } + if runtime.ran("docker load") { + t.Errorf("the image was loaded again although the machine held it: %v", runtime.commands) + } + if !strings.Contains(strings.Join(said, "\n"), "already held") { + t.Errorf("nothing was said about finding the image already there: %v", said) + } +} + +// A load that reported success and left nothing there is a failure, not a convergence +// (novox/hq ADR 0018). Without the read-back it would surface later as the host refusing a bundle +// naming an image nothing serves — a true message about the wrong thing. +func TestALoadThatLeftNothingBehindIsAFailure(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if len(args) > 1 && args[0] == "image" && args[1] == "inspect" { + return "", errors.New("Error: No such image") + } + return "Loaded image: mesh-control:test\n", nil + }} + + _, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), false, func(string) {}) + if err == nil { + t.Fatal("a load that left nothing on the machine was reported as success") + } + // Named by the tag, because that is what was asked about and what is missing. + if !strings.Contains(err.Error(), "mesh-control:test") { + t.Errorf("the failure does not say what this machine holds nothing of: %v", err) + } +} + +// **A dry run cannot know the id, and says so rather than pretending.** It loads nothing, so no +// runtime has decided anything, and the id in the archive is a fact about a file rather than a +// prediction about this machine. `--dry-run` still produces and checks the bundle's shape; what it +// cannot promise is the one value that only a load can settle. +func TestADryRunLoadsNothingAndSaysTheIdIsUnconfirmed(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if len(args) > 1 && args[0] == "image" && args[1] == "inspect" { + return "", errors.New("Error: No such image") + } + return "", fmt.Errorf("a dry run ran %v", args) + }} + + var said []string + loaded, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), true, func(line string) { said = append(said, line) }) + if err != nil { + t.Fatal(err) + } + if !loaded.Predicted { + t.Error("a dry run reported an id no runtime had confirmed as though it had been") + } + if loaded.ID != "sha256:"+fixtureDigest { + t.Errorf("a dry run reported %q, and the archive says sha256:%s", loaded.ID, fixtureDigest) + } + if runtime.ran("docker load") { + t.Errorf("a dry run loaded an image: %v", runtime.commands) + } + if !strings.Contains(strings.Join(said, "\n"), "NOT THE FINAL ID") { + t.Errorf("a dry run did not say its id is unconfirmed: %v", said) + } +} + +// A dry run on a machine that already holds the image DOES know the id, because the runtime was +// asked and answered. Reading is not changing, so a dry run is entitled to that. +func TestADryRunOnAMachineThatHoldsItKnowsTheRealId(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if len(args) > 1 && args[0] == "image" && args[1] == "inspect" { + return "sha256:" + runtimeDigest + "\n", nil + } + return "", fmt.Errorf("a dry run ran %v", args) + }} + + loaded, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), true, func(string) {}) + if err != nil { + t.Fatal(err) + } + if loaded.Predicted { + t.Error("an id this machine's runtime supplied was reported as a prediction") + } + if loaded.ID != "sha256:"+runtimeDigest { + t.Errorf("the id is %q, and the runtime said sha256:%s", loaded.ID, runtimeDigest) + } +} + +// **An untagged archive is a build-time fault, refused rather than worked around.** Without a tag +// there is no portable name to ask the runtime about, and the only thing left is scraping the +// sentence `docker load` prints for a person — which differs between runtime versions and is +// exactly the kind of guess this whole step exists to stop making. +func TestAnUntaggedArchiveIsRefused(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + return "", fmt.Errorf("nothing should have been run: %v", args) + }} + + _, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest, []string{}...), false, func(string) {}) + if err == nil { + t.Fatal("an archive with no tag was accepted, and there is no way to ask about it") + } + if !strings.Contains(err.Error(), "make bootstrap") { + t.Errorf("the refusal does not say how to build one that is tagged: %v", err) + } + if len(runtime.commands) != 0 { + t.Errorf("the machine was touched first: %v", runtime.commands) + } +} + +// An installer built from a plain checkout carries no image, and says which build step is missing. +// Discovered here, before a machine is touched, rather than after a bundle has been written. +func TestAnInstallerCarryingNoImageSaysSoRatherThanRaisingHalfAMesh(t *testing.T) { + if !image.IsEmpty() { + t.Skip("this checkout has a saved image embedded") + } + _, err := Load(context.Background(), (&asked{}).run, false, func(string) {}) + if !errors.Is(err, image.ErrEmpty) { + t.Fatalf("an installer with no control-plane image gave %v, want ErrEmpty", err) + } + if !strings.Contains(err.Error(), "make bootstrap") { + t.Errorf("the refusal does not say how to build one that carries an image: %v", err) + } +} + +// An answer that is not an image id is refused rather than written into a bundle. +// +// **This replaces a test that refused an answer differing from the archive's id.** That test +// encoded the mistake: a differing id is now the expected case, not a fault, because a runtime +// rewrites an image's configuration as it loads. What is still worth refusing is an answer that is +// not an id at all — that value becomes the name a bundle applies on a machine with no mesh to +// check anything against, so it is checked where the refusal can say whose mistake it is. +func TestARuntimeAnsweringSomethingThatIsNotAnImageIdIsRefused(t *testing.T) { + for _, nonsense := range []string{ + "mesh-control:test", + "sha256:" + strings.Repeat("9", 63), + "", + } { + runtime := &asked{answer: func(_ string, _ []string) (string, error) { + return nonsense + "\n", nil + }} + if _, err := loadImage(context.Background(), runtime.run, + savedImageFixture(t, fixtureDigest), false, func(string) {}); err == nil { + t.Errorf("the runtime answered %q and it was accepted as an image id", nonsense) + } + } +} diff --git a/internal/bootstrap/module.go b/internal/bootstrap/module.go new file mode 100644 index 0000000..2c6eb43 --- /dev/null +++ b/internal/bootstrap/module.go @@ -0,0 +1,145 @@ +package bootstrap + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" +) + +// placeholderDigest is what the catalogue writes where a built image's digest will go. +// +// Sixty-four zeros. A manifest in the catalogue names an image the mesh builds and pushes, and +// until that has happened there is no digest to name — so the convention is a digest that is +// obviously not one, replaced by the pipeline when it publishes. The installer meets it twice: it +// must NOT be there in the registry's manifest, whose image is upstream and never built +// (novox/hq 04-ISSUES/029), and it MUST be there in the control plane's, which is the image +// step 8 has just pushed. +const placeholderDigest = "sha256:" + "0000000000000000000000000000000000000000000000000000000000000000" + +// catalogueDir is where a mesh-catalog checkout keeps its manifests. +const catalogueDir = "modules" + +// manifestFile is the path a module's manifest is read from, given a catalogue checkout. +func manifestFile(catalogue, module string) string { + return filepath.Join(catalogue, catalogueDir, module, "module.json") +} + +// readManifest reads one module's manifest out of a mesh-catalog checkout. +// +// **From a checkout rather than from anything the mesh serves**, and that is the ordering the whole +// pivot exists to respect: at this point the mesh has no build machine, no forge and — until step 7 +// finishes — no registry. What a manifest is, is a file; the installer is handed the directory it +// is in and reads it. +func readManifest(catalogue, module string) ([]byte, error) { + path := manifestFile(catalogue, module) + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf( + "the %s module's manifest could not be read: %w\n"+ + "--catalog is a checkout of the mesh's catalogue repository, and this is read from "+ + "%s inside it", module, err, filepath.Join(catalogueDir, module, "module.json")) + } + return raw, nil +} + +// Installed is what installing one module did. +type Installed struct { + // Module is its name. + Module string + // Known is true when the mesh already had this module in its catalogue. The manifest is + // registered either way — a re-run with a changed manifest must land — so this reports what + // was found rather than what was skipped. + Known bool + // Assigned is true when this node was given the module during this run. + Assigned bool + // Pushed is what the mesh said when it sent this node its declaration. + Pushed string +} + +// installModule registers a manifest, gives it to this node, and sends it. +// +// The three verbs a person types, in the order they type them, through the same commands. There is +// no installer-only path into the mesh: everything here is `module add`, `assign` and `push`, so +// what the installer does on a bare machine and what an operator does on a running mesh are the +// same act (novox/hq ADR 0035, one refusal per act however it is asked for). +func installModule(ctx context.Context, o Options, control controlPlane, module string, + manifest []byte, say func(string)) (Installed, error) { + + out, err := registerAndAssign(ctx, o, control, module, manifest, say) + if err != nil { + return out, err + } + out.Pushed, err = pushNode(ctx, o, control, say) + return out, err +} + +// registerAndAssign is the half of installing that happens before anything is sent. +// +// Split out because one module needs something in between: the control plane's own store +// connections have to be accepted before its declaration is composed, or the mesh would seal +// thirty-two random bytes into the file it expects a connection string in and the container would +// come up unable to open anything (mesh-control's `secret accept`, and what it exists for). +// +// **`module add` is run every time and is not skipped when the module is already known.** It is an +// upsert on the manifest, and the manifest is exactly what changes between runs — step 9 registers +// the control plane with a digest that did not exist the first time. Skipping it because the name +// was already in the catalogue would silently pin the mesh to the previous image. +func registerAndAssign(ctx context.Context, o Options, control controlPlane, module string, + manifest []byte, say func(string)) (Installed, error) { + + out := Installed{Module: module} + + known, err := control.tell(ctx, "module", "list") + if err != nil { + return out, err + } + out.Known = mentions(known, module) + + remote := "/" + module + "-module.json" + if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil { + return out, err + } + if _, err := control.tell(ctx, "module", "add", remote); err != nil { + return out, err + } + if out.Known { + say(" registered " + module + " — the mesh already knew it; the manifest is now this one") + } else { + say(" registered " + module) + } + + assigned, err := control.tell(ctx, "assign", o.Node, module) + if err != nil { + return out, err + } + out.Assigned = true + say(" assigned " + module + " to " + o.Node) + if refusal := strings.TrimSpace(assigned); strings.Contains(refusal, "but ") { + // `assign` records what a person meant and says at once when the machine cannot host it. + // Repeated rather than swallowed: the push below will apply everything else and this is + // the only place the reason appears. + say(indent(refusal)) + } + + return out, nil +} + +// pushNode sends this node everything it should be. +// +// Given the long wait rather than the probe timeout: a push composes every declaration this node +// should hold, seals every secret in them and publishes them, and on a first node that is the +// slowest thing the mesh does. +func pushNode(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) { + said, err := control.within(o.Wait).tell(ctx, "push", o.Node) + if err != nil { + return "", fmt.Errorf( + "%w\n\nWhat was registered and assigned is registered and assigned, and this node has "+ + "not been sent it. Nothing is half-applied — a push the mesh refused sent nothing "+ + "at all. Fix what it named and run this installer again: it will find the module "+ + "already registered and try the push again", err) + } + say(" pushed " + o.Node) + return strings.TrimSpace(said), nil +} diff --git a/internal/bootstrap/preflight.go b/internal/bootstrap/preflight.go new file mode 100644 index 0000000..4c0bb21 --- /dev/null +++ b/internal/bootstrap/preflight.go @@ -0,0 +1,240 @@ +package bootstrap + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/image" + "github.com/novox/mesh-host/internal/profile" +) + +// DefaultRegistry is where an image reference that names no host comes from. +const DefaultRegistry = "registry-1.docker.io:443" + +// Preflight refuses early and plainly, and returns the bundle template it read. +// +// Everything here is a thing that will otherwise be discovered half way through: a machine with +// no runtime found after a bundle has been written, a template that does not parse found after an +// image has been loaded, a registry that cannot be reached found inside a `docker pull` that +// reports a network error and not a missing image. The order is cheapest first, so a mistake in +// what the installer was pointed at costs nothing to find. +func Preflight(ctx context.Context, o Options, d Deps, say func(string)) ([]byte, error) { + // 1. Does this installer carry what it claims to? + // + // Asked before the machine is touched, for the same reason `mesh-host bundle` exists: a host + // that carries no substrate must say so when somebody asks, not on a first node + // (internal/bundle). An installer built without an image would otherwise get a machine as far + // as a running store and a running broker and stop. + if image.IsEmpty() { + return nil, image.ErrEmpty + } + saved, err := image.Saved() + if err != nil { + return nil, err + } + // + // The id said here is the ARCHIVE's, and it is reported as such: it is a fact about the file + // and not about this machine. What this runtime will call the image once it holds it is the + // runtime's decision, made at the load, and asked for there (see Load). + carriedID, err := image.ArchiveID(saved) + if err != nil { + return nil, err + } + tag := firstOr(image.Tags(saved), "") + if tag == "" { + // Refused here as well as at the load, because preflight's whole job is to find at the + // start what would otherwise be found half way through — and this would be found after an + // image had been written to disk and handed to a container runtime. + return nil, fmt.Errorf( + "the carried control-plane image has no tag, and the installer identifies it by one: "+ + "an image id is the digest of a configuration that a runtime rewrites as it loads, "+ + "so the archive's id (%s) is not necessarily the id this machine would hold.\n"+ + "Rebuild the installer with a tagged image: `make bootstrap IMAGE=:`", + carriedID) + } + say(fmt.Sprintf(" control plane %s carried (the archive calls it %s)", tag, carriedID)) + + // 2. Is the template there, and is it a substrate? + template, err := os.ReadFile(o.Template) + if err != nil { + return nil, fmt.Errorf( + "the bundle template could not be read: %w\n"+ + "It is what this machine will be asked to be, so there is nothing to do without "+ + "it. Point --bundle at one; mesh-host's examples/substrate-first-node.lock is "+ + "the shape", err) + } + // Parsed here as well as at the rewrite, because a template that is not a declaration should + // cost a second rather than an image load and a written file. + parsed, err := declaration.ParseFileTrusted(template) + if err != nil { + return nil, fmt.Errorf("the bundle template is not a declaration: %w", err) + } + if _, err := controlPlaneIn(parsed); err != nil { + return nil, err + } + say(fmt.Sprintf(" bundle template %s (%d resources)", o.Template, len(parsed.Resources))) + + // 3. Does a container runtime ANSWER? + // + // Not "is it installed" — novox/hq 04-ISSUES/007 is exactly that mistake, and the detector + // this uses is the one written for it: it asks the daemon for its server version, which fails + // when the daemon is down however complete the installation is. + // + // **Yes, the bundle installs the runtime itself**, and that is not a contradiction. The + // installer needs one BEFORE the apply, because the control plane's image is loaded into it + // first; the bundle still declares the package and the service because the host must own them + // and reassert them at every reconcile. So this is not a duplicate check — it is the one thing + // the bootstrap cannot bootstrap. + // + // Polled rather than asked once. A socket-activated daemon queued behind + // `network-online.target` is not absent, it is a few seconds away, and `docker load` against + // one blocks silently rather than failing (04-ISSUES/024). Waiting is the honest reading. + if err := waitForRuntime(ctx, d.Run, o.Timeout, o.Wait, say); err != nil { + return nil, err + } + + // 4. Can this machine reach what the bundle's images come from? + // + // Asked of the hosts the bundle actually names rather than of the internet in general. The + // mesh's own image is carried and needs nothing served — it is skipped here for exactly that + // reason. Everything else is somebody else's image at somebody else's registry, and a machine + // that cannot reach it fails inside a pull, which reports a network error where a person + // reads a missing image. + // + // "The mesh's own image is skipped" used to mean "skipped if the template happened to name it + // in a way that needs no registry", and that is not the same sentence. A template names the + // control plane by SOMETHING — the reference is a slot, and step 3 replaces whatever is in it + // with the id of the image this installer carries. Whatever the slot held is therefore never + // pulled, never fetched, and never reached; requiring it to be reachable refuses a correct + // install because of a string that is about to be thrown away. Found on the first real run: the + // lab's template still carried `192.0.2.250:5000/mesh-control@…`, the address of a registry that + // no longer exists, and preflight timed out dialling it. + for _, host := range registriesIn(parsed) { + dialing, cancel := context.WithTimeout(ctx, o.Timeout) + err := d.Dial(dialing, host) + cancel() + if err != nil { + return nil, fmt.Errorf( + "this machine cannot reach %s, and the bundle's images are served from there: "+ + "%w\nThe apply would fail inside a pull, which says the wrong thing. Fix the "+ + "machine's network, or point the bundle at a registry it can reach", + host, err) + } + say(" reachable " + host) + } + return template, nil +} + +// waitForRuntime asks the runtime, repeatedly, until it answers or the wait runs out. +func waitForRuntime(ctx context.Context, run Runner, probe, wait time.Duration, say func(string)) error { + detector := containerRuntimeDetector(run) + + deadline := time.Now().Add(wait) + var last string + for { + probing, cancel := context.WithTimeout(ctx, probe) + verdict := detector.Detect(probing) + cancel() + if verdict.Present { + say(" container runtime " + verdict.Detail) + return nil + } + last = verdict.Detail + + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(runtimeAskEvery): + } + } + return fmt.Errorf( + "this machine has no container runtime that answers, after waiting %s: %s\n"+ + "An installed package is not a capability (novox/hq 04-ISSUES/007) — the daemon was "+ + "asked and did not reply. Start it, then run this again; every step is idempotent", + wait, last) +} + +// runtimeAskEvery is how often the runtime is asked again while waiting for it. +var runtimeAskEvery = 2 * time.Second + +// containerRuntimeDetector is the host's OWN detector for a working runtime, not a second +// implementation of the same question. Two answers to "is there a container runtime here" is how +// the installer and the host come to disagree about a machine. +func containerRuntimeDetector(run Runner) profile.Detector { + for _, detector := range profile.Default(profile.Runner(run)) { + if detector.Name() == profile.CapContainerRuntime { + return detector + } + } + // Unreachable unless the host's own detector set loses its container runtime, which would be + // a change nobody would make on purpose — said rather than nil-dereferenced. + panic("the host detects no container runtime capability, and the installer needs that answer") +} + +// registriesIn is every host the bundle's images would be fetched from, without duplicates and in +// the order they appear. +// +// The control plane's own resource is excluded by identity rather than by the shape of what it +// names. Its image reference is a slot the installer overwrites with the id of the image it +// carries, so no registry ever serves it — and a template that filled that slot with a registry +// this machine cannot reach is not a machine with a network problem. +func registriesIn(d *declaration.Declaration) []string { + var hosts []string + seen := map[string]bool{} + for _, r := range d.Resources { + container, ok := r.(*declaration.Container) + if !ok || container.Identity() == ControlPlaneID { + continue + } + host, served := registryOf(container.Image) + if !served || seen[host] { + continue + } + seen[host] = true + hosts = append(hosts, host) + } + return hosts +} + +// registryOf says where an image would be fetched from, and whether anything has to serve it. +// +// The second return is false for an image named by the digest of its own configuration: nothing +// serves those and nothing can (see `internal/declaration`'s checkImage). That is the whole reason +// the mesh's own control plane can be raised on a machine with no registry anywhere. +// +// The rule for the rest is the container runtime's own: the part before the first slash is a +// registry host if it looks like one — it has a dot, or a port, or it is `localhost` — and +// otherwise it is part of a repository name on the default registry. +func registryOf(reference string) (string, bool) { + if reference == "" || strings.HasPrefix(reference, "sha256:") { + return "", false + } + name := reference + if at := strings.Index(name, "@"); at >= 0 { + name = name[:at] + } + + first, _, hasPath := strings.Cut(name, "/") + if !hasPath || !(strings.Contains(first, ".") || strings.Contains(first, ":") || first == "localhost") { + return DefaultRegistry, true + } + if !strings.Contains(first, ":") { + // A registry with no port is reached over HTTPS, which is where a pull would go. + return first + ":443", true + } + return first, true +} + +func firstOr(values []string, fallback string) string { + if len(values) == 0 || strings.TrimSpace(values[0]) == "" { + return fallback + } + return values[0] +} diff --git a/internal/bootstrap/preflight_test.go b/internal/bootstrap/preflight_test.go new file mode 100644 index 0000000..1664da1 --- /dev/null +++ b/internal/bootstrap/preflight_test.go @@ -0,0 +1,151 @@ +package bootstrap + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +// An installed package is not a capability (novox/hq 04-ISSUES/007). The daemon is asked, and a +// machine where it does not answer is refused before anything is loaded, written or applied. +// +// The refusal has to be plain, because the person reading it is standing in front of a machine +// that will not work: it says what was asked, what came back, that re-running is safe, and names +// the record that explains why an installed docker is not enough. +func TestPreflightRefusesPlainlyWhenTheRuntimeDoesNotAnswer(t *testing.T) { + silent := func(context.Context, string, ...string) (string, error) { + return "", errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock") + } + + // No wait, so this is one attempt: what is being tested is the refusal, not the patience. + err := waitForRuntime(context.Background(), silent, time.Second, 0, func(string) {}) + if err == nil { + t.Fatal("a machine whose container runtime does not answer was accepted") + } + for _, wanted := range []string{ + "no container runtime that answers", + "Cannot connect to the Docker daemon", + "04-ISSUES/007", + "idempotent", + } { + if !strings.Contains(err.Error(), wanted) { + t.Errorf("the refusal does not mention %q:\n%v", wanted, err) + } + } +} + +// And a runtime that is merely slow to start is waited for rather than refused. +// +// A socket-activated daemon queued behind the network is not absent, it is a few seconds away. +// Refusing on the first attempt would make a correct bootstrap fail for being observed too early — +// and `docker load` against such a daemon blocks silently rather than failing, which is how one +// became a 35-minute silence (04-ISSUES/024). +func TestARuntimeThatIsStillStartingIsWaitedFor(t *testing.T) { + previous := runtimeAskEvery + runtimeAskEvery = time.Millisecond + defer func() { runtimeAskEvery = previous }() + + attempts := 0 + slow := func(context.Context, string, ...string) (string, error) { + attempts++ + if attempts < 3 { + return "", errors.New("Cannot connect to the Docker daemon") + } + return "27.0.3\n", nil + } + + var said []string + if err := waitForRuntime(context.Background(), slow, time.Second, time.Second, + func(line string) { said = append(said, line) }); err != nil { + t.Fatalf("a runtime that answered on the third ask was refused: %v", err) + } + if attempts != 3 { + t.Errorf("the runtime was asked %d time(s)", attempts) + } + if !strings.Contains(strings.Join(said, "\n"), "27.0.3") { + t.Errorf("the version the daemon reported was not said back: %v", said) + } +} + +// What has to be reachable is what the bundle actually names, not "the internet". +// +// The mesh's own image is carried and nothing serves it, so asking a registry about it would be +// asking a question with no answer — which is the whole point of naming an image by the digest of +// its own configuration. +func TestOnlyTheRegistriesTheBundleNamesAreAskedAbout(t *testing.T) { + parsed, err := declaration.ParseFileTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"store","type":"container","name":"mesh-store","image":"postgres@sha256:` + + strings.Repeat("7", 64) + `"}, + {"id":"broker","type":"container","name":"mesh-broker","image":"192.0.2.250:5000/lavinmq@sha256:` + + strings.Repeat("8", 64) + `"}, + {"id":"control-plane","type":"container","name":"mesh-control","image":"` + held + `"} + ]}`)) + if err != nil { + t.Fatal(err) + } + + got := registriesIn(parsed) + want := []string{DefaultRegistry, "192.0.2.250:5000"} + if len(got) != len(want) { + t.Fatalf("asked about %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("asked about %v, want %v", got, want) + } + } +} + +// And the control plane's slot is excluded whatever is in it. +// +// Found on the first real run of this installer. A template names the control plane by SOMETHING +// and step 3 replaces it with the id of the carried image, so whatever was there is never pulled — +// but preflight was reading that slot like any other and dialling it. The lab's template still +// carried the address of a registry the lab no longer raises, so a correct install timed out in +// preflight against a machine with a perfectly good network. +func TestTheControlPlanesOwnRegistryIsNeverAskedAbout(t *testing.T) { + parsed, err := declaration.ParseFileTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"store","type":"container","name":"mesh-store","image":"postgres@sha256:` + + strings.Repeat("7", 64) + `"}, + {"id":"control-plane","type":"container","name":"mesh-control","image":"192.0.2.250:5000/mesh-control@sha256:` + + strings.Repeat("8", 64) + `"} + ]}`)) + if err != nil { + t.Fatal(err) + } + + got := registriesIn(parsed) + if len(got) != 1 || got[0] != DefaultRegistry { + t.Fatalf("asked about %v; the control plane's own reference is about to be replaced and "+ + "must not be reached for", got) + } +} + +func TestWhereAnImageWouldBeFetchedFrom(t *testing.T) { + // The container runtime's own rule: the part before the first slash is a registry host if it + // has a dot, a port, or is localhost. Getting this wrong means dialling a hostname that is + // really the first half of a repository name, and refusing a machine that is fine. + for _, c := range []struct { + reference string + host string + served bool + }{ + {"postgres@sha256:" + strings.Repeat("a", 64), DefaultRegistry, true}, + {"cloudamqp/lavinmq@sha256:" + strings.Repeat("a", 64), DefaultRegistry, true}, + {"192.0.2.250:5000/postgres@sha256:" + strings.Repeat("a", 64), "192.0.2.250:5000", true}, + {"localhost/mesh-control@sha256:" + strings.Repeat("a", 64), "localhost:443", true}, + {"registry.example.com/a/b@sha256:" + strings.Repeat("a", 64), "registry.example.com:443", true}, + // Held by this machine. Nothing serves it, and nothing can. + {"sha256:" + strings.Repeat("a", 64), "", false}, + {"", "", false}, + } { + host, served := registryOf(c.reference) + if host != c.host || served != c.served { + t.Errorf("%q → (%q, %v), want (%q, %v)", c.reference, host, served, c.host, c.served) + } + } +} diff --git a/internal/bootstrap/publish.go b/internal/bootstrap/publish.go new file mode 100644 index 0000000..e7288b1 --- /dev/null +++ b/internal/bootstrap/publish.go @@ -0,0 +1,173 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" +) + +// ControlPlaneRepository is what the control plane's image is called in the mesh's own registry. +const ControlPlaneRepository = "mesh-control" + +// genesisTag is the tag the first push uses. +// +// A tag is not a pin and is never what anything is deployed from — the digest the registry assigns +// is (novox/hq ADR 0006). This exists so a person reading `/v2/mesh-control/tags/list` can see +// which image this mesh started from, and so the push has something to name. Everything downstream +// uses the digest that comes back. +const genesisTag = "genesis" + +// Published is what step 8 did. +type Published struct { + // Reference is `/mesh-control@sha256:…` — the first manifest digest this image has + // ever had, and the thing that makes the control plane an ordinary module. + Reference string + // Tagged is where it was pushed, tag and all. + Tagged string + // Already is true when the registry was already serving it and nothing was pushed. + Already bool +} + +// PublishControlPlane puts the carried image into the mesh's own registry and reads back its digest. +// +// **This is the pivot's hinge.** Every image must be pinned by digest, and a digest a pin can mean +// is one a REGISTRY assigned when something was pushed to it. The control plane's image is built +// from source and pushed nowhere, so it has none — which is why the substrate names it by the +// digest of its own configuration, and why that is legal exactly where nothing could have served +// one. The moment this push completes, that stops being true: the image has a manifest digest, so +// the control plane can be named the way every other module is named, so the mesh can build and +// roll out its own upgrades. If this step is skipped the machine still works and the mesh cannot +// upgrade itself, which is the check novox/hq ADR 0067 states: after installing, the running +// control plane must be pinned by a digest the mesh's own registry assigned, not by an image id. +// +// **It mirrors mesh-control's `internal/builder`.PublishImage rather than importing it.** Tag, +// push, read back `RepoDigests`, refuse anything without `@sha256:` — the same four steps, because +// there is exactly one right way to learn what a registry will serve something as, and it is to +// ask the registry. It is not imported because that code is tier 2: the host and its installer +// depend on nothing that must be installed first (novox/hq ADR 0041), and taking a dependency on +// the control plane's repository to raise the control plane would be the cycle this whole ADR is +// about, one layer up. The duplication is four commands, and it is deliberate. +// +// One difference, and it is a correction rather than a divergence: the digest is chosen from +// `RepoDigests` by repository instead of taken as element zero. An image that has been pushed to +// more than one registry has more than one entry, and element zero is then whichever the runtime +// happened to list first — which would pin this mesh to somebody else's registry, silently. +func PublishControlPlane(ctx context.Context, o Options, d Deps, imageID string, + say func(string)) (Published, error) { + + remote := o.Registry + "/" + ControlPlaneRepository + out := Published{Tagged: remote + ":" + genesisTag} + + // Asked first. A digest already served is a fact about the registry, and re-pushing an image + // the registry already holds is asking it to store what it already has under the name it + // already has. + if held, err := digestOf(ctx, o, d, remote); err != nil { + return out, err + } else if held != "" { + out.Reference, out.Already = held, true + say(" already published " + held) + return out, nil + } + + if _, err := d.Run(ctx, "docker", "tag", imageID, out.Tagged); err != nil { + return out, fmt.Errorf("cannot tag the carried image as %s: %w", out.Tagged, err) + } + if _, err := d.Run(ctx, "docker", "push", out.Tagged); err != nil { + return out, fmt.Errorf( + "the container runtime would not push %s: %w\n"+ + "The registry is plain HTTP and wants no credentials, deliberately — it is reached "+ + "over the mesh's own network, which is already the encrypted and authenticated "+ + "thing. A runtime refusing it for being insecure is refusing a registry on %s, "+ + "which it does not do for a loopback address", + out.Tagged, err, o.Registry) + } + + // Read back, from the registry's own answer rather than computed here. What matters is what + // the registry will serve for that reference, and only it can say (novox/hq ADR 0018). + pinned, err := digestOf(ctx, o, d, remote) + if err != nil { + return out, err + } + if pinned == "" { + return out, fmt.Errorf( + "%s was pushed and the registry does not serve it.\n"+ + "The next step names the control plane's module by the digest this was supposed to "+ + "produce, so there is nothing to name. Check `docker push` and "+ + "http://%s/v2/%s/tags/list", out.Tagged, o.Registry, ControlPlaneRepository) + } + out.Reference = pinned + say(" published " + pinned) + return out, nil +} + +// digestOf is what the registry serves this repository as, or empty if it serves it at all. +// +// Both halves are asked, because either alone lies. The registry's tag list says something was +// pushed and not what its digest is; the runtime's `RepoDigests` says what a digest was and not +// whether the registry still has it — a registry whose volume was recreated would leave the +// runtime remembering a digest nothing serves, and the module registered against it would pin the +// mesh to an image that cannot be pulled. +func digestOf(ctx context.Context, o Options, d Deps, remote string) (string, error) { + asking, cancel := context.WithTimeout(ctx, o.Timeout) + status, body, err := d.Fetch(asking, + "http://"+o.Registry+"/v2/"+ControlPlaneRepository+"/tags/list") + cancel() + if err != nil { + return "", fmt.Errorf("cannot ask the registry at %s what it holds: %w", o.Registry, err) + } + if status == http.StatusNotFound { + // Nothing has ever been pushed under this name. An answer, not a failure. + return "", nil + } + if status != http.StatusOK { + return "", fmt.Errorf("the registry answered %d when asked what it holds for %s", + status, ControlPlaneRepository) + } + var listed struct { + Tags []string `json:"tags"` + } + if err := json.Unmarshal([]byte(body), &listed); err != nil { + return "", fmt.Errorf("the registry's answer about %s is not readable: %w", + ControlPlaneRepository, err) + } + if !contains(listed.Tags, genesisTag) { + return "", nil + } + + // The registry has it. What digest, according to the runtime that pushed it. + reading, cancel := context.WithTimeout(ctx, o.Timeout) + out, err := d.Run(reading, "docker", "inspect", "--format", "{{json .RepoDigests}}", + remote+":"+genesisTag) + cancel() + if err != nil { + // The registry holds the tag and this machine's runtime does not hold the image. That + // happens on a re-run after the image was pruned, and it is not something to work around + // by trusting the tag: a tag can be made to point elsewhere. + return "", nil + } + var digests []string + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &digests); err != nil { + return "", fmt.Errorf("the runtime's answer about %s is not readable: %w", remote, err) + } + for _, digest := range digests { + if !strings.HasPrefix(digest, remote+"@sha256:") { + // Somebody else's registry serving the same image. Skipped rather than used: pinning + // this mesh's control plane to a registry it does not run is exactly the dependency + // the pivot exists to remove. + continue + } + return digest, nil + } + return "", nil +} + +func contains(values []string, want string) bool { + for _, v := range values { + if v == want { + return true + } + } + return false +} diff --git a/internal/bootstrap/publish_test.go b/internal/bootstrap/publish_test.go new file mode 100644 index 0000000..759414d --- /dev/null +++ b/internal/bootstrap/publish_test.go @@ -0,0 +1,197 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "testing" + "time" +) + +// Step 8 is the pivot's hinge (novox/hq ADR 0067): the carried image gets a manifest digest, which +// is the first one it has ever had, and that is what lets the control plane be named the way every +// other module is named. These tests defend how that digest is learned, because a wrong one pins +// the mesh to an image nothing on this machine serves. + +func publishing(t *testing.T, fetch func(string) (int, string, error), + run func(name string, args []string) (string, error)) (Options, Deps, *asked) { + t.Helper() + runtime := &asked{answer: run} + return Options{ + Registry: "127.0.0.1:5000", + Timeout: time.Second, + Wait: 0, + }, Deps{ + Run: runtime.run, + Fetch: func(_ context.Context, url string) (int, string, error) { + return fetch(url) + }, + }, runtime +} + +// Nothing has ever been pushed under this name, so the registry says 404 — and that is an answer, +// not a failure. An installer that treated it as one would refuse on the first run of the step it +// exists to perform. +func TestAnImageNoRegistryHasEverHeldIsPushed(t *testing.T) { + pushed := false + o, d, runtime := publishing(t, + func(string) (int, string, error) { + if !pushed { + return http.StatusNotFound, "", nil + } + return http.StatusOK, `{"name":"mesh-control","tags":["genesis"]}`, nil + }, + func(_ string, args []string) (string, error) { + switch args[0] { + case "tag": + return "", nil + case "push": + pushed = true + return "", nil + case "inspect": + return `["127.0.0.1:5000/mesh-control@sha256:` + strings.Repeat("a", 64) + `"]`, nil + } + return "", fmt.Errorf("unexpected: %v", args) + }) + + out, err := PublishControlPlane(context.Background(), o, d, held, func(string) {}) + if err != nil { + t.Fatal(err) + } + if out.Already { + t.Error("an image no registry held was reported as already published") + } + if !strings.HasPrefix(out.Reference, "127.0.0.1:5000/mesh-control@sha256:") { + t.Errorf("the control plane is pinned as %q", out.Reference) + } + if !runtime.ran("docker push 127.0.0.1:5000/mesh-control:genesis") { + t.Errorf("nothing was pushed: %v", runtime.commands) + } +} + +// An image the registry already serves is not pushed again, and says so. Blobs are named by their +// content, so re-pushing is asking a registry to store what it already has under the name it +// already has — and the installer is run over and over. +func TestAnImageTheRegistryAlreadyServesIsNotPushedAgain(t *testing.T) { + o, d, runtime := publishing(t, + func(string) (int, string, error) { + return http.StatusOK, `{"name":"mesh-control","tags":["genesis"]}`, nil + }, + func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return `["127.0.0.1:5000/mesh-control@sha256:` + strings.Repeat("b", 64) + `"]`, nil + } + return "", fmt.Errorf("unexpected: %v", args) + }) + + out, err := PublishControlPlane(context.Background(), o, d, held, func(string) {}) + if err != nil { + t.Fatal(err) + } + if !out.Already { + t.Error("an image the registry already serves was not reported as already published") + } + if runtime.ran("docker push") { + t.Errorf("it was pushed again: %v", runtime.commands) + } +} + +// **The digest is chosen by repository, not taken as element zero.** An image that has been pushed +// to more than one registry has more than one entry, and element zero is whichever the runtime +// listed first — which would pin this mesh's control plane to somebody else's registry, silently, +// which is the dependency the whole pivot exists to remove. +func TestTheDigestComesFromThisMeshsOwnRegistry(t *testing.T) { + elsewhere := "some.other.registry/mesh-control@sha256:" + strings.Repeat("c", 64) + ours := "127.0.0.1:5000/mesh-control@sha256:" + strings.Repeat("d", 64) + + o, d, _ := publishing(t, + func(string) (int, string, error) { + return http.StatusOK, `{"tags":["genesis"]}`, nil + }, + func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return `["` + elsewhere + `","` + ours + `"]`, nil + } + return "", fmt.Errorf("unexpected: %v", args) + }) + + out, err := PublishControlPlane(context.Background(), o, d, held, func(string) {}) + if err != nil { + t.Fatal(err) + } + if out.Reference != ours { + t.Errorf("the control plane is pinned as %q, and this mesh's registry serves %q", + out.Reference, ours) + } +} + +// A push that produced no digest this mesh's registry serves is refused, and the refusal says what +// depends on it. The next step names the control plane's module by that digest, so there would be +// nothing to name — and finding that out one step later would mean registering a module pinned to +// an empty string. +func TestAPushThatProducedNoDigestIsRefused(t *testing.T) { + pushed := false + o, d, _ := publishing(t, + func(string) (int, string, error) { + if !pushed { + return http.StatusNotFound, "", nil + } + // Pushed, and the registry still does not list it. + return http.StatusOK, `{"tags":[]}`, nil + }, + func(_ string, args []string) (string, error) { + if args[0] == "push" { + pushed = true + } + return "", nil + }) + + _, err := PublishControlPlane(context.Background(), o, d, held, func(string) {}) + if err == nil { + t.Fatal("a push that produced no digest was accepted") + } + if !strings.Contains(err.Error(), "does not serve it") { + t.Errorf("the refusal does not say what is missing: %v", err) + } +} + +// A tag is not a pin. If the runtime answers with something that is not pinned by digest, it is +// not used — a tag can be made to point at a different image, and this reference is applied on +// machines with no mesh to ask about anything (novox/hq ADR 0006). +func TestATagIsNotAPin(t *testing.T) { + o, d, _ := publishing(t, + func(string) (int, string, error) { + return http.StatusOK, `{"tags":["genesis"]}`, nil + }, + func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return `["127.0.0.1:5000/mesh-control:genesis"]`, nil + } + return "", nil + }) + + out, err := PublishControlPlane(context.Background(), o, d, held, func(string) {}) + if err == nil { + t.Fatalf("a tag was accepted as a pin: %q", out.Reference) + } +} + +// A registry that cannot be reached at all is said so plainly rather than becoming a push that +// fails for a reason nobody can read. +func TestARegistryThatCannotBeAskedIsSaidSo(t *testing.T) { + o, d, _ := publishing(t, + func(string) (int, string, error) { + return 0, "", errors.New("connection refused") + }, + func(string, []string) (string, error) { return "", nil }) + + _, err := PublishControlPlane(context.Background(), o, d, held, func(string) {}) + if err == nil { + t.Fatal("a registry that refused the connection was treated as empty") + } + if !strings.Contains(err.Error(), "cannot ask the registry") { + t.Errorf("the refusal does not say the registry could not be asked: %v", err) + } +} diff --git a/internal/bootstrap/registry.go b/internal/bootstrap/registry.go new file mode 100644 index 0000000..637d9e7 --- /dev/null +++ b/internal/bootstrap/registry.go @@ -0,0 +1,204 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" +) + +// RegistryModule is the module that provides the mesh's artifact store. +const RegistryModule = "registry" + +// registryResource is the id of the resource in that module's manifest that runs the registry. +// What the container is CALLED is read from the manifest rather than assumed, because the name is +// the catalogue's to choose and the installer only has to know which resource to wait for. +const registryResource = "store" + +// Registry is what step 7 did. +type Registry struct { + Installed + // Container is the container the manifest declares, confirmed running. + Container string + // Address is host:port the registry answers on, as this machine reaches it. + Address string + // Answered is the status the registry's own `/v2/` gave back. + Answered int +} + +// InstallRegistry gives this mesh somewhere to put images. +// +// **Its image is upstream and it is never built.** novox/hq 04-ISSUES/029 is the whole reason this +// step exists in this position: a module that provides the artifact store cannot be delivered +// through the artifact store, so the registry is the one module whose image is pulled from the +// internet like the store and the broker before it. A manifest carrying the catalogue's +// placeholder digest here would mean somebody had made it buildable, which is the cycle again — +// so it is refused rather than pulled. +// +// **No credentials, and that is deliberate.** The registry is reached over the mesh's own private +// network, which is already the encrypted and authenticated thing; a second layer inside it would +// be certificates to issue and rotate for no property the first does not have (mesh-control's +// `internal/builder`, which pushes to it the same way). So there is nothing here to configure and +// nothing to seal — which is also why step 8 can push without the mesh having issued anything. +// +// **It is verified by asking it, not by looking at it.** A container that is up is not a registry +// that serves: `/v2/` is the registry API's own "yes, I am one and I am ready", and it is the +// question step 8 depends on the answer to. +func InstallRegistry(ctx context.Context, o Options, d Deps, control controlPlane, + say func(string)) (Registry, error) { + + out := Registry{Address: o.Registry} + + manifest, err := readManifest(o.Catalogue, RegistryModule) + if err != nil { + return out, err + } + container, image, err := containerIn(manifest, registryResource) + if err != nil { + return out, err + } + if strings.Contains(image, placeholderDigest) { + return out, fmt.Errorf( + "the %s module's image is %q, which is the catalogue's placeholder for something the "+ + "mesh builds and pushes.\n"+ + "This module is the one that cannot work that way: it PROVIDES the place built "+ + "images go, so it can never be delivered through it (novox/hq 04-ISSUES/029). Its "+ + "image is upstream and pinned in the manifest", RegistryModule, image) + } + out.Container = container + say(" registry image " + image + " — upstream, never built") + + installed, err := installModule(ctx, o, control, RegistryModule, manifest, say) + out.Installed = installed + if err != nil { + return out, err + } + + // Read back, in two stages, because they fail differently. A container that never appears is + // a declaration that did not reach this node or an image that would not pull; a container + // that is up and does not answer is a registry that started and failed. + if err := waitForContainer(ctx, control.run, o.Timeout, o.Wait, container, say); err != nil { + return out, err + } + status, err := waitForTheRegistry(ctx, d, o, say) + if err != nil { + return out, err + } + out.Answered = status + return out, nil +} + +// waitForTheRegistry asks `/v2/` until it answers. +func waitForTheRegistry(ctx context.Context, d Deps, o Options, say func(string)) (int, error) { + where := "http://" + o.Registry + "/v2/" + + deadline := time.Now().Add(o.Wait) + var last string + for { + asking, cancel := context.WithTimeout(ctx, o.Timeout) + status, _, err := d.Fetch(asking, where) + cancel() + switch { + case err != nil: + last = err.Error() + case status == http.StatusOK: + say(fmt.Sprintf(" replies %s answered %d", where, status)) + return status, nil + default: + // A registry that answers 401 is one that wants credentials, which this one is + // configured not to. Reported as what it said rather than retried into a timeout. + last = fmt.Sprintf("it answered %d", status) + } + + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + return 0, ctx.Err() + case <-time.After(answerEvery): + } + } + return 0, fmt.Errorf( + "the registry's container is running and %s does not answer, after waiting %s: %s\n"+ + "Running is not serving. `/v2/` is the registry API saying it is ready, and the next "+ + "step pushes the control plane's image to it — so this is refused here rather than "+ + "discovered inside a `docker push`. `docker logs mesh-registry` says what it did", + where, o.Wait, last) +} + +// waitForContainer waits for a container the mesh was asked to create to be running. +// +// Unlike the substrate's own verify, this one waits: the mesh applies through a node's host, over +// the broker, asynchronously. A push that the control plane accepted has not yet happened on the +// machine, and refusing on the first look would refuse every correct install. +func waitForContainer(ctx context.Context, run Runner, probe, wait time.Duration, name string, + say func(string)) error { + + deadline := time.Now().Add(wait) + var last string + for { + state, err := containerRunning(ctx, run, probe, name) + switch { + case err != nil: + last = "it is not there at all" + case state.running: + say(" running " + name) + return nil + default: + last = "it is " + state.status + } + + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(answerEvery): + } + } + return fmt.Errorf( + "the mesh accepted the push and %q is not running after %s: %s\n"+ + "The control plane sends a declaration over the broker and this node's host applies "+ + "it, so the two ends fail differently: `mesh-host` on this machine says what it made "+ + "of the declaration, and `status` on the control plane says whether it was collected "+ + "at all", name, wait, last) +} + +// containerIn finds one container resource in a module manifest and gives back its name and image. +// +// It reads the manifest as data rather than through the catalogue's own parser, because that +// parser lives in the control plane and the host depends on nothing installed first +// (novox/hq ADR 0041) — importing it would put tier 2 inside tier 0. What is read here is two +// fields of a shape the catalogue owns; the manifest is handed to the control plane unchanged, and +// it is the control plane's `module add` that judges whether it is a manifest at all. +func containerIn(manifest []byte, id string) (name, image string, err error) { + var m struct { + Module string `json:"module"` + Resources []struct { + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + Image string `json:"image"` + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + return "", "", fmt.Errorf("this manifest is not readable as JSON: %w", err) + } + var containers []string + for _, r := range m.Resources { + if r.Type != "container" { + continue + } + containers = append(containers, r.ID) + if r.ID == id { + return r.Name, r.Image, nil + } + } + return "", "", fmt.Errorf( + "the %s module declares no container %q, so the installer does not know what to wait for. "+ + "It declares: %s", m.Module, id, strings.Join(containers, ", ")) +} diff --git a/internal/bootstrap/registry_test.go b/internal/bootstrap/registry_test.go new file mode 100644 index 0000000..52f923d --- /dev/null +++ b/internal/bootstrap/registry_test.go @@ -0,0 +1,216 @@ +package bootstrap + +import ( + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// Step 7 installs the one module whose image can never come from the mesh's own registry, because +// it IS the mesh's own registry (novox/hq 04-ISSUES/029). These tests defend that, and defend the +// distinction the whole verify layer of this program is built on: a container that is up is not a +// service that answers. + +// catalogueWith writes a fake catalogue checkout holding one module's manifest. +// +// A fixture here rather than the real catalogue, unlike the substrate example the rewrite tests +// use: the catalogue is a different repository on a different branch, and a test that read it +// would pass or fail according to what somebody else had checked out. +func catalogueWith(t *testing.T, module, manifest string) string { + t.Helper() + root := t.TempDir() + dir := filepath.Join(root, catalogueDir, module) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + return root +} + +const upstreamRegistryManifest = `{ + "module": "registry", + "version": "1", + "provides": [{"name": "artifact-store", "scope": "mesh"}], + "capabilities": ["container-runtime"], + "resources": [ + {"id": "state", "type": "directory", "path": "/var/lib/mesh/registry", "mode": "0700"}, + {"id": "store", "type": "container", "name": "mesh-registry", + "image": "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373", + "ports": ["5000:5000"]} + ] +}` + +// aMeshThatAgrees answers every command the installer issues at steps 7 and 9 the way a working +// mesh would, except for whatever a test overrides. +func aMeshThatAgrees(answers map[string]string) func(string, []string) (string, error) { + return func(name string, args []string) (string, error) { + joined := strings.Join(args, " ") + for fragment, said := range answers { + if strings.Contains(joined, fragment) { + return said, nil + } + } + switch { + case name != "docker": + return "", fmt.Errorf("unexpected program %q", name) + case args[0] == "cp": + return "", nil + case args[0] == "inspect": + return "true running\n", nil + case args[0] == "exec": + return "", nil + } + return "", fmt.Errorf("unexpected: %v", args) + } +} + +func installing(t *testing.T, catalogue string) Options { + t.Helper() + return Options{ + Node: "anchor", + Catalogue: catalogue, + Registry: "127.0.0.1:5000", + Timeout: time.Second, + Wait: 0, + } +} + +// A container that is up is not a registry that serves. `/v2/` is the registry API's own "yes, I +// am one and I am ready", and the step after this pushes to it — so it is refused here rather than +// discovered inside a `docker push`. +func TestARegistryContainerThatIsUpIsNotARegistryThatServes(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + runtime := &asked{answer: aMeshThatAgrees(nil)} + deps := Deps{ + Run: runtime.run, + Fetch: func(context.Context, string) (int, string, error) { + return http.StatusInternalServerError, "", nil + }, + } + + _, err := InstallRegistry(context.Background(), + installing(t, catalogueWith(t, RegistryModule, upstreamRegistryManifest)), + deps, controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err == nil { + t.Fatal("a registry whose container is up and which answers 500 was accepted") + } + for _, wanted := range []string{"/v2/", "Running is not serving"} { + if !strings.Contains(err.Error(), wanted) { + t.Errorf("the refusal does not mention %q:\n%v", wanted, err) + } + } +} + +// The whole of step 7, against a mesh that agrees: registered, assigned, pushed, up, and answering. +func TestARegistryThatAnswersIsAccepted(t *testing.T) { + runtime := &asked{answer: aMeshThatAgrees(nil)} + deps := Deps{ + Run: runtime.run, + Fetch: func(context.Context, string) (int, string, error) { + return http.StatusOK, "{}", nil + }, + } + + out, err := InstallRegistry(context.Background(), + installing(t, catalogueWith(t, RegistryModule, upstreamRegistryManifest)), + deps, controlPlane{container: "temp-mesh-control", run: runtime.run, timeout: time.Second}, + func(string) {}) + if err != nil { + t.Fatal(err) + } + if out.Container != "mesh-registry" { + t.Errorf("the registry's container is %q", out.Container) + } + if out.Answered != http.StatusOK { + t.Errorf("the registry answered %d", out.Answered) + } + // Registered, assigned and pushed, through the same three commands a person types. + for _, wanted := range []string{ + "module add /registry-module.json", + "assign anchor registry", + "push anchor", + } { + if !runtime.ran(wanted) { + t.Errorf("the installer never ran %q: %v", wanted, runtime.commands) + } + } +} + +// **The registry's image is upstream and it is never built.** A manifest carrying the catalogue's +// placeholder digest would mean somebody had made this module buildable — which is the cycle +// novox/hq 04-ISSUES/029 settled: a module that provides the artifact store cannot be delivered +// through the artifact store. +func TestARegistryManifestThatWantsBuildingIsRefused(t *testing.T) { + wants := strings.Replace(upstreamRegistryManifest, + "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373", + "mesh-runtime-registry@"+placeholderDigest, 1) + + runtime := &asked{answer: aMeshThatAgrees(nil)} + _, err := InstallRegistry(context.Background(), + installing(t, catalogueWith(t, RegistryModule, wants)), + Deps{Run: runtime.run}, controlPlane{container: "temp-mesh-control", run: runtime.run}, + func(string) {}) + if err == nil { + t.Fatal("a registry manifest naming an image the mesh would have to build was accepted") + } + if !strings.Contains(err.Error(), "04-ISSUES/029") { + t.Errorf("the refusal does not name the decision it rests on: %v", err) + } + if runtime.ran("module add") { + t.Error("it was registered anyway") + } +} + +// A catalogue that is not there is said plainly, with what --catalog is. This is the most likely +// mistake anybody makes at this step and the least interesting to debug. +func TestACatalogueThatIsNotThereIsSaidPlainly(t *testing.T) { + runtime := &asked{answer: aMeshThatAgrees(nil)} + _, err := InstallRegistry(context.Background(), + installing(t, filepath.Join(t.TempDir(), "nowhere")), + Deps{Run: runtime.run}, controlPlane{container: "temp-mesh-control", run: runtime.run}, + func(string) {}) + if err == nil { + t.Fatal("a catalogue that does not exist was accepted") + } + if !strings.Contains(err.Error(), "--catalog") { + t.Errorf("the refusal does not say what to fix: %v", err) + } +} + +// A refusal from the control plane is repeated verbatim. mesh-control refuses in paragraphs — +// "nothing provides route, wanted by registry" — and an installer that reported "exit status 1" +// would throw away the only thing a person can act on. +func TestWhatTheMeshRefusedIsRepeated(t *testing.T) { + refusal := "nothing provides \"route\", wanted by registry" + runtime := &asked{answer: func(name string, args []string) (string, error) { + if strings.Contains(strings.Join(args, " "), "push") { + return refusal, fmt.Errorf("exit status 1") + } + return aMeshThatAgrees(nil)(name, args) + }} + + _, err := InstallRegistry(context.Background(), + installing(t, catalogueWith(t, RegistryModule, upstreamRegistryManifest)), + Deps{Run: runtime.run}, controlPlane{container: "temp-mesh-control", run: runtime.run, + timeout: time.Second}, func(string) {}) + if err == nil { + t.Fatal("a push the mesh refused was reported as successful") + } + if !strings.Contains(err.Error(), refusal) { + t.Errorf("what the mesh said is not in the failure:\n%v", err) + } + if !strings.Contains(err.Error(), "run this installer again") { + t.Errorf("the failure does not say a re-run continues from here:\n%v", err) + } +} diff --git a/internal/bootstrap/retire.go b/internal/bootstrap/retire.go new file mode 100644 index 0000000..a84055d --- /dev/null +++ b/internal/bootstrap/retire.go @@ -0,0 +1,291 @@ +package bootstrap + +import ( + "bytes" + "context" + "fmt" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/system" +) + +// Retired is what step 10 did. +type Retired struct { + // Container is the temporary control plane that was dropped. + Container string + // Gone is true when the machine no longer has it. + Gone bool + // Already is true when it was gone before this step ran. + Already bool + // Bundle is where the bundle without it was written. + Bundle string + // Removed is how many resources the re-apply reported removing. + Removed int +} + +// RetireTheTemporaryControlPlane drops it from the bundle and lets the host take it away. +// +// **Destruction by omission, which is the host's ordinary behaviour and not a new mechanism.** The +// host owns what it has applied and removes what it owns and is no longer declared. So retiring the +// temporary control plane is not a verb anybody had to invent: the bundle stops declaring it, the +// bundle is applied again, and the removal pass does what it does for every other resource that +// leaves a declaration. +// +// That is the whole of why the rename at step 3 mattered. Had the substrate and the module both +// called their container `mesh-control`, this apply would have removed the module's container — +// the host would have been asked to take away something it believed it owned, and it would have +// been right. Two names, two owners, and the removal is unambiguous. +// +// **It is the last step for a reason.** Until step 9 has a control plane that answers, the +// temporary one is the only thing that can tell this machine anything, and a machine left with no +// control plane cannot be fixed remotely (novox/hq ADR 0067). So this runs after the permanent one +// has been proved, and a run interrupted before it leaves two control planes, which is untidy and +// harmless — a re-run reaches this step and finishes. +func RetireTheTemporaryControlPlane(ctx context.Context, o Options, sys system.System, + produced []byte, run Runner, say func(string)) (Retired, error) { + + out := Retired{Bundle: o.Out} + + current, err := declaration.ParseFileTrusted(produced) + if err != nil { + return out, fmt.Errorf("the bundle this installer produced is not a declaration: %w", err) + } + temporary, err := controlPlaneIn(current) + if err != nil { + // The bundle already declares no control plane, which is what a re-run after this step + // finds. Nothing to drop, and nothing to be alarmed about. + say(" already dropped the bundle declares no temporary control plane") + out.Already = true + return out, nil + } + out.Container = temporary.Name + + // Textual, for the reason the rewrite at step 3 is textual: the produced bundle is meant to be + // READ, and a person coming to a machine after a pivot should be able to open the file the + // installer applied and see the substrate they recognise with the control plane gone from it. + // Re-serialising a parsed declaration would drop every comment in it. + bundle, err := removeResource(produced, ControlPlaneID) + if err != nil { + return out, err + } + without, err := declaration.ParseFileTrusted(bundle) + if err != nil { + return out, fmt.Errorf( + "taking the temporary control plane out of the bundle broke it: %w", err) + } + if _, err := controlPlaneIn(without); err == nil { + return out, fmt.Errorf( + "the bundle still declares %q after it was taken out, so nothing was removed and the "+ + "apply below would change nothing", ControlPlaneID) + } + if err := writeBundleFile(o.Out, bundle); err != nil { + return out, err + } + say(fmt.Sprintf(" wrote %s (%d resources) — without %s", + o.Out, len(without.Resources), temporary.Name)) + + // Applied the same way everything else here is applied, under the same origin, against the + // same state file. What makes this a removal rather than a no-op is that the state file + // records the container as something this installer applied, and the declaration no longer + // asks for it. + report, err := ApplyBundle(ctx, o, sys, without, run, say) + if err != nil { + return out, fmt.Errorf( + "%w\n\nThe permanent control plane is running and the temporary one is still here. "+ + "That is untidy and it is not broken: two control planes on one mesh are both "+ + "stateless and both correct. Run this installer again to finish", err) + } + for _, outcome := range report.Outcomes { + if outcome.Action == "removed" { + out.Removed++ + } + } + + // Read back. A removal that reported success and left the container running would leave two + // control planes consuming the same broker queues for ever, which is the state this step + // exists to end. + gone, err := isGone(ctx, run, o.Timeout, o.Wait, temporary.Name) + if err != nil { + return out, err + } + out.Gone = gone + if !gone { + return out, fmt.Errorf( + "the apply reported the temporary control plane removed and %q is still running.\n"+ + "Two control planes are consuming this mesh's broker queues. Neither is wrong and "+ + "the mesh is not damaged, but the pivot is not finished: `docker rm -f %s` ends "+ + "it, and this installer will then agree", temporary.Name, temporary.Name) + } + say(" gone " + temporary.Name) + return out, nil +} + +// removeResource takes one resource out of a bundle's text, comments and all. +// +// It walks the `resources` array counting braces, skipping over strings and comments so that a +// `//` inside a connection string is not read as the start of one — the substrate's own bundle +// contains `postgres://…` several times, and a scanner that did not know the difference would +// treat the rest of the line as a comment and lose a brace. +// +// What is removed is the element AND whatever precedes it back to the previous element, which is +// where the comment explaining it lives. A comment that outlives the thing it describes is worse +// than no comment: it is the file telling somebody the machine has a control plane it does not. +func removeResource(bundle []byte, id string) ([]byte, error) { + array := indexOutsideStrings(bundle, `"resources"`) + if array < 0 { + return nil, fmt.Errorf("this bundle has no resources array, so there is nothing to take out of it") + } + open := indexOutsideStrings(bundle[array:], "[") + if open < 0 { + return nil, fmt.Errorf("this bundle's resources are not a list") + } + open += array + + depth, from := 0, -1 + previous := open + inString, escaped, inLine, inBlock := false, false, false, false + for i := open + 1; i < len(bundle); i++ { + c := bundle[i] + switch { + case escaped: + escaped = false + case inString && c == '\\': + escaped = true + case inString: + if c == '"' { + inString = false + } + case inLine: + if c == '\n' { + inLine = false + } + case inBlock: + if c == '*' && i+1 < len(bundle) && bundle[i+1] == '/' { + inBlock, i = false, i+1 + } + case c == '"': + inString = true + case c == '/' && i+1 < len(bundle) && bundle[i+1] == '/': + inLine, i = true, i+1 + case c == '/' && i+1 < len(bundle) && bundle[i+1] == '*': + inBlock, i = true, i+1 + case c == '{': + if depth == 0 { + from = i + } + depth++ + case c == '}': + depth-- + if depth != 0 { + break + } + if isResource(bundle[from:i+1], id) { + return cut(bundle, previous, from, i+1), nil + } + previous = i + 1 + from = -1 + case c == ']' && depth == 0: + return nil, fmt.Errorf( + "this bundle declares no %q, so there is nothing to take out of it", id) + } + } + return nil, fmt.Errorf("this bundle's resources list does not end") +} + +// isResource reports whether one resource's text is the one wanted. +// +// Whitespace-insensitive on the pair, quotes included, so `"id": "control-plane"` and +// `"id":"control-plane"` are the same answer and `"id": "control-planes"` is not. +func isResource(resource []byte, id string) bool { + var tight []byte + for _, c := range resource { + if c != ' ' && c != '\t' && c != '\n' && c != '\r' { + tight = append(tight, c) + } + } + return bytes.Contains(tight, []byte(`"id":"`+id+`"`)) +} + +// cut removes an element and what leads up to it, leaving the list valid. +// +// Whether the comma before or the comma after goes depends on where the element sits: an element +// with something before it takes the comma that joined them, and the first element takes the one +// after it. Getting this wrong produces a trailing comma, which is JSON nothing will parse — +// caught by the re-parse either way, and better not produced. +func cut(bundle []byte, previous, from, to int) []byte { + start := from + for i := previous; i < from; i++ { + if bundle[i] == ',' { + start = i + break + } + } + end := to + if start == from { + // Nothing before it, so the comma that follows is the one that would be left dangling. + for i := to; i < len(bundle); i++ { + if bundle[i] == ',' { + end = i + 1 + break + } + if bundle[i] == ']' { + break + } + } + } + out := make([]byte, 0, len(bundle)) + out = append(out, bundle[:start]...) + return append(out, bundle[end:]...) +} + +// indexOutsideStrings finds a fragment that is not inside a JSON string. +func indexOutsideStrings(haystack []byte, needle string) int { + inString, escaped := false, false + for i := 0; i < len(haystack); i++ { + switch { + case escaped: + escaped = false + continue + case haystack[i] == '\\' && inString: + escaped = true + continue + case haystack[i] == '"': + // The needle may itself start with a quote, so the match is tried before the quote is + // consumed. + if !inString && bytes.HasPrefix(haystack[i:], []byte(needle)) { + return i + } + inString = !inString + continue + case inString: + continue + } + if bytes.HasPrefix(haystack[i:], []byte(needle)) { + return i + } + } + return -1 +} + +// isGone waits for a container to stop existing. +// +// Waited for rather than asked once, because a container being removed is a container that is +// stopping first, and a runtime answers about it until it has finished. +func isGone(ctx context.Context, run Runner, probe, wait time.Duration, name string) (bool, error) { + deadline := time.Now().Add(wait) + for { + if _, err := containerRunning(ctx, run, probe, name); err != nil { + // The runtime does not know it. That is the answer being waited for. + return true, nil + } + if time.Now().After(deadline) { + return false, nil + } + select { + case <-ctx.Done(): + return false, ctx.Err() + case <-time.After(answerEvery): + } + } +} diff --git a/internal/bootstrap/retire_test.go b/internal/bootstrap/retire_test.go new file mode 100644 index 0000000..efdec44 --- /dev/null +++ b/internal/bootstrap/retire_test.go @@ -0,0 +1,165 @@ +package bootstrap + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +// Retirement is destruction by omission, which is the host's ordinary behaviour: it owns what it +// applied and removes what it owns and is no longer declared. These tests defend the bundle +// surgery that expresses it, because a bundle that came out of it unparseable would be found by +// the apply — after the file on the machine had already been replaced. + +func produced(t *testing.T) []byte { + t.Helper() + out, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + return out.Bundle +} + +// The temporary control plane leaves the bundle, everything else stays, and what is left parses. +func TestTheTemporaryControlPlaneLeavesTheBundleAndNothingElseDoes(t *testing.T) { + before, err := declaration.ParseFileTrusted(produced(t)) + if err != nil { + t.Fatal(err) + } + shorter, err := removeResource(produced(t), ControlPlaneID) + if err != nil { + t.Fatal(err) + } + after, err := declaration.ParseFileTrusted(shorter) + if err != nil { + t.Fatalf("the bundle without the control plane does not parse: %v\n%s", err, shorter) + } + if len(after.Resources) != len(before.Resources)-1 { + t.Fatalf("the bundle went from %d resources to %d, and one was removed", + len(before.Resources), len(after.Resources)) + } + if _, err := controlPlaneIn(after); err == nil { + t.Error("the bundle still declares a control plane") + } + // The store and the broker are still exactly what they were. A retirement that took the + // substrate with it would leave the machine with a module and nothing under it. + for id, name := range containerNames(before) { + if id == ControlPlaneID { + continue + } + if containerNames(after)[id] != name { + t.Errorf("%s was lost or renamed by the retirement", id) + } + } +} + +// **A `//` inside a string is not a comment.** The substrate's own bundle carries +// `postgres://…` several times, and a scanner that read the rest of those lines as a comment +// would lose braces and cut the wrong thing out — silently, because what it produced would still +// look like a file. +func TestASchemeInsideAStringIsNotReadAsAComment(t *testing.T) { + shorter, err := removeResource(produced(t), ControlPlaneID) + if err != nil { + t.Fatal(err) + } + after, err := declaration.ParseFileTrusted(shorter) + if err != nil { + t.Fatal(err) + } + // The migration action, which is the resource holding the most `://` of anything here, is + // still whole. + found := false + for _, r := range after.Resources { + if r.Identity() == "context-schemas" { + found = true + } + } + if !found { + t.Error("the resource full of connection strings did not survive the removal") + } +} + +// Removing the FIRST element takes the comma after it rather than the comma before it, because +// there is no comma before it. Getting this wrong produces a leading comma, which is JSON nothing +// parses — and the file would already have been written. +func TestRemovingTheFirstResourceLeavesAValidList(t *testing.T) { + shorter, err := removeResource(produced(t), "container-runtime") + if err != nil { + t.Fatal(err) + } + after, err := declaration.ParseFileTrusted(shorter) + if err != nil { + t.Fatalf("removing the first resource broke the bundle: %v\n%s", err, shorter) + } + for _, r := range after.Resources { + if r.Identity() == "container-runtime" { + t.Error("the first resource is still there") + } + } +} + +// A bundle that declares no such resource is refused rather than silently returned unchanged. A +// removal that removed nothing and reported success would leave the apply below with nothing to +// do and the installer claiming a pivot it did not finish. +func TestRemovingSomethingThatIsNotThereIsRefused(t *testing.T) { + if _, err := removeResource(produced(t), "nothing-of-the-sort"); err == nil { + t.Fatal("a bundle was reported to have had a resource removed that it never declared") + } +} + +// A re-run after the retirement finds a bundle with no control plane in it and says so, rather +// than failing. This is the idempotence of the last step, and it is the one a person is most +// likely to exercise: the pivot ends here, so a re-run to check ends here too. +func TestRetiringABundleThatAlreadyHasNoControlPlaneIsAlreadyDone(t *testing.T) { + shorter, err := removeResource(produced(t), ControlPlaneID) + if err != nil { + t.Fatal(err) + } + var said []string + out, err := RetireTheTemporaryControlPlane(context.Background(), Options{}, + nil, shorter, nil, func(line string) { said = append(said, line) }) + if err != nil { + t.Fatal(err) + } + if !out.Already { + t.Error("a bundle with no control plane in it was not reported as already retired") + } + if !strings.Contains(strings.Join(said, "\n"), "already dropped") { + t.Errorf("the run does not say it was already done: %v", said) + } +} + +// A container the runtime still knows about after the apply is a pivot that did not finish. Two +// control planes on one mesh are both correct and neither is wrong — but the temporary one was +// supposed to go, and saying it went when it did not is the fault this project keeps naming. +func TestAContainerStillThereAfterRemovalIsNotGone(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + stillThere := &asked{answer: func(_ string, _ []string) (string, error) { + return "true running\n", nil + }} + gone, err := isGone(context.Background(), stillThere.run, time.Second, 0, "temp-mesh-control") + if err != nil { + t.Fatal(err) + } + if gone { + t.Error("a container the runtime still describes was reported gone") + } + + removed := &asked{answer: func(_ string, _ []string) (string, error) { + return "", errors.New("No such object: temp-mesh-control") + }} + gone, err = isGone(context.Background(), removed.run, time.Second, 0, "temp-mesh-control") + if err != nil { + t.Fatal(err) + } + if !gone { + t.Error("a container the runtime does not know about was not reported gone") + } +} diff --git a/internal/bootstrap/rewrite.go b/internal/bootstrap/rewrite.go new file mode 100644 index 0000000..dc040e4 --- /dev/null +++ b/internal/bootstrap/rewrite.go @@ -0,0 +1,336 @@ +package bootstrap + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/novox/mesh-host/internal/declaration" +) + +// ControlPlaneID is the resource the installer replaces the image of. +// +// A resource id rather than a container name or a guess at the image, because the id is the one +// thing a declaration promises is stable — it is what lets the store say *this is the same +// resource I applied last time* (`internal/declaration`, Resource.Identity). A bundle that does not +// name one is refused rather than applied without a control plane, which would raise a store and a +// broker and no mesh. +const ControlPlaneID = "control-plane" + +// TempPrefix is what the substrate's control plane is renamed with. +// +// **This is the whole of how a carried resource becomes a declared one.** The substrate raises a +// control plane and a module later declares one, and for a moment both exist — which looked like a +// handover problem needing a way for the host to stop owning something without destroying it. It is +// not one. The temporary control plane is called `temp-mesh-control` and the permanent one is +// called `mesh-control`: two containers, two owners, nothing shared and nothing to hand over. At +// the end the temporary one is dropped from the bundle and the host removes it, which is exactly +// what should happen to something named "temp" (novox/hq ADR 0067). +// +// The name is also the audit. After the pivot, a machine running `mesh-control` and not +// `temp-mesh-control` has completed it; one running both stopped in the middle; one running only +// the temp has not started. That is readable from `docker ps` by somebody who knows nothing else. +const TempPrefix = "temp-" + +// ControlPlaneModule is the module the permanent control plane is installed as, and the name its +// container takes — the name the substrate's own control plane gives up here so that it can. +// +// Declared beside the rename rather than beside the step that uses it, because this is where the +// two names are decided together and where the reason for both of them is written down. +const ControlPlaneModule = "mesh-control" + +// brokerAddressVar is what a token tells an enrolling node to dial. +// +// Not rewritten here — see the note in Rewrite — but reported, because it is the field most likely +// to be wrong on a machine that is not the one the template was written for, and it is wrong in a +// way nothing notices until a second node tries to join. +const brokerAddressVar = "MESH_BROKER_ADDRESS" + +// Rewritten is the bundle this machine will apply, and what was done to produce it. +type Rewritten struct { + // Bundle is the produced file's bytes — the template with one image reference replaced, + // comments and all. + Bundle []byte + // Declaration is that bundle, parsed. Carried so the apply and the verify are talking about + // the same document rather than each re-reading the file and hoping. + Declaration *declaration.Declaration + + // Was is the image the template named the control plane by; Now is the one it names it by. + Was string + Now string + // Places is how many times that reference appeared, and therefore how many were replaced. + Places int + // Changed is false when the template already named this image — a re-run on a bundle this + // installer produced earlier. + Changed bool + + // WasCalled is what the template called the control plane's container; TempName is what the + // produced bundle calls it. Renamed is false when the template already used the temporary name. + WasCalled string + TempName string + Renamed bool + + // Kept is every other container image, unchanged, as " ". Reported rather than + // assumed: "postgres was left alone" is a claim, and this is the evidence for it. + Kept []string + + // Resources is how many things the produced bundle asks for. + Resources int + // BrokerAddress is what the control plane will tell enrolling nodes to dial, or empty. + BrokerAddress string +} + +// Rewrite produces the bundle this machine will apply from the template it was given. +// +// **Two substitutions, and both are textual.** The control plane's image becomes the id of the image +// this machine now holds, and its container is renamed `temp-mesh-control`; nothing else changes. +// The rename is what makes the pivot expressible at all — see TempPrefix. Textual rather than +// parse-and-re-serialise because +// the produced file has to be *read* — a person getting a machine working must be able to open it, +// see the substrate they recognise, and see exactly one thing different. Re-serialising a parsed +// declaration would drop every comment in the template, and those comments are where the reasons +// live. +// +// **Every place that reference appears, not only the container.** The bundle names the control +// plane's image twice: once as the container that runs `serve`, and once inside the action that +// runs `migrate` to create the contexts' schemas. Replacing only the container would leave the +// migration pointing at an image no registry serves, and the apply would fail in the middle — +// after the store is up, before the broker. They are one image and they move together. +// +// **Third-party images are not touched.** postgres and lavinmq keep the `name@sha256:` references +// the template carries and are pulled from wherever those name (novox/hq ADR 0006). This is +// checked afterwards rather than merely intended: the produced bundle is re-parsed and every other +// container's image is compared against what it was. +// +// **What this deliberately does NOT rewrite:** MESH_BROKER_ADDRESS, the endpoint every enrolment +// token will carry. It differs per machine and it is silently fatal when wrong — a node enrols +// against a dead address and nothing complains until it fails to come back. It belongs to the +// enrolment stage, which is not built yet, so this reports it loudly and leaves it alone rather +// than guessing an address for a machine it has not been told about. +func Rewrite(template []byte, imageID string) (Rewritten, error) { + if !isImageID(imageID) { + return Rewritten{}, fmt.Errorf( + "%q is not an image id. The control plane is named by the digest of its own "+ + "configuration — sha256: and sixty-four hex characters — because nothing serves "+ + "it and there is no manifest digest to use instead", imageID) + } + + before, err := declaration.ParseFileTrusted(template) + if err != nil { + return Rewritten{}, fmt.Errorf("the bundle template is not a declaration: %w", err) + } + control, err := controlPlaneIn(before) + if err != nil { + return Rewritten{}, err + } + + out := Rewritten{Was: control.Image, Now: imageID, BrokerAddress: control.Env[brokerAddressVar]} + + occurrences := bytes.Count(template, []byte(control.Image)) + if occurrences == 0 { + // The parser found the image and the bytes do not contain it, which means the two are + // reading different things. Refused rather than replaced-zero-times-and-reported-success. + return Rewritten{}, fmt.Errorf( + "the control plane's image is %q according to the parsed template, and that text is "+ + "not in the file. Nothing was rewritten", control.Image) + } + out.Places = occurrences + + switch { + case control.Image == imageID: + // Already this image. The idempotent case, and the one that happens whenever somebody + // re-runs the installer against a bundle it produced earlier. + out.Bundle = template + default: + out.Bundle = bytes.ReplaceAll(template, []byte(control.Image), []byte(imageID)) + out.Changed = true + } + + // And the container is renamed, for the reason TempPrefix records. Done here rather than + // anywhere later because the bundle is the only place the name is decided: the apply creates + // the container from it, the verify asks that container questions, and the retirement takes + // this same resource back out. One name, one place, read by everything. + out.WasCalled, out.TempName = control.Name, TempPrefix+control.Name + if strings.HasPrefix(control.Name, TempPrefix) { + out.TempName = control.Name + } + renamed, err := renameContainer(out.Bundle, control.Name, out.TempName) + if err != nil { + return Rewritten{}, err + } + out.Bundle, out.Renamed = renamed, out.TempName != control.Name + + // Read back, on the bytes that will actually be applied. Everything above is an intention + // until the produced file is parsed and asked what it says. + after, err := declaration.ParseFileTrusted(out.Bundle) + if err != nil { + return Rewritten{}, fmt.Errorf( + "the bundle this produced is not a declaration, so the substitution broke it: %w", err) + } + out.Declaration, out.Resources = after, len(after.Resources) + + produced, err := controlPlaneIn(after) + if err != nil { + return Rewritten{}, err + } + if produced.Image != imageID { + return Rewritten{}, fmt.Errorf( + "the produced bundle still names the control plane %q, not %q", produced.Image, imageID) + } + if produced.Name != out.TempName { + return Rewritten{}, fmt.Errorf( + "the produced bundle still calls the control plane's container %q, not %q. The "+ + "permanent one is a module and takes the plain name, so a substrate that kept it "+ + "would put two owners on one container", produced.Name, out.TempName) + } + + // And nothing else moved. A substitution on text can in principle catch more than it was + // aimed at, and "postgres was left exactly as it was" is the claim this checks rather than + // asserts. Both the image AND the name, because there are now two substitutions. + wasImage, wasName := containerImages(before), containerNames(before) + for id, image := range containerImages(after) { + if id == ControlPlaneID { + continue + } + if wasImage[id] != image { + return Rewritten{}, fmt.Errorf( + "rewriting the control plane's image also changed %q, from %q to %q. Only the "+ + "mesh's own image may move; everything else is somebody else's image at "+ + "somebody else's registry", id, wasImage[id], image) + } + out.Kept = append(out.Kept, fmt.Sprintf("%s %s", id, image)) + } + for id, name := range containerNames(after) { + if id == ControlPlaneID || wasName[id] == name { + continue + } + return Rewritten{}, fmt.Errorf( + "renaming the control plane's container also renamed %q, from %q to %q. Only the "+ + "control plane moves out of the way; every other container keeps the name the "+ + "substrate gave it", id, wasName[id], name) + } + sortStrings(out.Kept) + return out, nil +} + +// renameContainer changes one container's name in the bundle's text. +// +// **The quoted name, not the bare word.** `mesh-control` also appears inside the image reference +// the template carries (`…/mesh-control@sha256:…`) and could appear inside a command line; a bare +// substitution would catch those too. What is wanted is a JSON string that IS the name, so the +// quotes are part of what is matched — `"mesh-control"` matches the container's `name` and an +// action's `in`, which are exactly the places the name means the container, and nothing else. +// +// It refuses when the text does not contain what the parse says is there, for the same reason the +// image substitution does: the two would then be reading different things, and a rename that +// replaced nothing and reported success would leave the module and the substrate fighting over one +// container three steps later. +func renameContainer(bundle []byte, from, to string) ([]byte, error) { + if from == to { + return bundle, nil + } + quoted := []byte(`"` + from + `"`) + if bytes.Count(bundle, quoted) == 0 { + return nil, fmt.Errorf( + "the control plane's container is called %q according to the parsed template, and %s "+ + "is not in the file. Nothing was renamed, and the substrate would raise a "+ + "container the module also wants", from, quoted) + } + return bytes.ReplaceAll(bundle, quoted, []byte(`"`+to+`"`)), nil +} + +// controlPlaneIn finds the container this installer replaces the image of. +func controlPlaneIn(d *declaration.Declaration) (*declaration.Container, error) { + for _, r := range d.Resources { + if r.Identity() != ControlPlaneID { + continue + } + container, ok := r.(*declaration.Container) + if !ok { + return nil, fmt.Errorf( + "this bundle's %q is a %s, and the control plane has to be a container for its "+ + "image to be named. Nothing was rewritten", ControlPlaneID, r.Kind()) + } + return container, nil + } + return nil, fmt.Errorf( + "this bundle names no %q, so there is no control plane to give this machine's image to. "+ + "A substrate without one raises a store and a broker and no mesh. It declares: %s", + ControlPlaneID, strings.Join(identities(d), ", ")) +} + +func containerImages(d *declaration.Declaration) map[string]string { + images := map[string]string{} + for _, r := range d.Resources { + if container, ok := r.(*declaration.Container); ok { + images[container.ID] = container.Image + } + } + return images +} + +func containerNames(d *declaration.Declaration) map[string]string { + names := map[string]string{} + for _, r := range d.Resources { + if container, ok := r.(*declaration.Container); ok { + names[container.ID] = container.Name + } + } + return names +} + +func identities(d *declaration.Declaration) []string { + var ids []string + for _, r := range d.Resources { + ids = append(ids, r.Identity()) + } + return ids +} + +// isImageID is the same shape `internal/declaration` accepts for an image the machine holds. Asked +// here as well so the refusal names the installer's own mistake, rather than surfacing as a +// declaration refusal about a bundle this program wrote. +func isImageID(s string) bool { + const prefix = "sha256:" + if !strings.HasPrefix(s, prefix) || len(s) != len(prefix)+64 { + return false + } + for _, c := range s[len(prefix):] { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +func sortStrings(values []string) { + for i := 1; i < len(values); i++ { + for j := i; j > 0 && values[j] < values[j-1]; j-- { + values[j], values[j-1] = values[j-1], values[j] + } + } +} + +// writeBundleFile puts the produced bundle where a person can read it, creating the directory it +// lives in. +// +// 0644, and that is deliberate: this file names an image and describes a substrate, and it holds +// the bootstrap credentials the template happens to carry — which are the same ones anybody can +// read in the template itself. It is meant to be read. What must not be world-readable is the +// node's identity, and that lives elsewhere and is written elsewhere (`internal/identity`). +func writeBundleFile(path string, content []byte) error { + if dir := filepath.Dir(path); dir != "" && dir != "." { + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("cannot make %s to write the produced bundle into: %w", dir, err) + } + } + if err := os.WriteFile(path, content, 0o644); err != nil { + return fmt.Errorf( + "cannot write the produced bundle to %s: %w\nIt is what is about to be applied, and "+ + "applying something nobody can read afterwards is how a machine becomes a mystery", + path, err) + } + return nil +} diff --git a/internal/bootstrap/rewrite_test.go b/internal/bootstrap/rewrite_test.go new file mode 100644 index 0000000..7902f1c --- /dev/null +++ b/internal/bootstrap/rewrite_test.go @@ -0,0 +1,321 @@ +package bootstrap + +import ( + "os" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" +) + +// Each test names the decision it defends (novox/hq ADR 0017). + +const ( + held = "sha256:1111111111111111111111111111111111111111111111111111111111111111" + otherHeld = "sha256:2222222222222222222222222222222222222222222222222222222222222222" +) + +// theRealBundle is this repository's own substrate example, used rather than a fixture. +// +// A fixture would agree with whatever this code does. The example is what an installer is actually +// pointed at, it names the control plane twice, and it is the file that changes when the substrate +// changes — so a rewrite that stops working on it is a rewrite that has stopped working. +func theRealBundle(t *testing.T) []byte { + t.Helper() + raw, err := os.ReadFile("../../examples/substrate-first-node.lock") + if err != nil { + t.Fatalf("reading the substrate example: %v", err) + } + return raw +} + +func TestTheControlPlaneIsNamedByTheImageThisMachineHolds(t *testing.T) { + out, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + if !out.Changed { + t.Error("the rewrite reported nothing changed, and the template named a registry image") + } + control, err := controlPlaneIn(out.Declaration) + if err != nil { + t.Fatal(err) + } + if control.Image != held { + t.Errorf("the control plane is %q, want %q", control.Image, held) + } +} + +// **Every place the bundle names that image, not only the container.** +// +// The substrate names the control plane's image twice: the container that runs `serve`, and the +// action that runs `migrate` to create the contexts' schemas. Rewriting only the container leaves +// the migration pointing at an image no registry serves, and the apply dies in the middle — after +// the store is up and before the broker. This is the test that would have caught that. +func TestEveryPlaceTheBundleNamesTheControlPlaneIsRewritten(t *testing.T) { + template := theRealBundle(t) + + out, err := Rewrite(template, held) + if err != nil { + t.Fatal(err) + } + if out.Places < 2 { + t.Fatalf("the control plane's image was found in %d place(s); the substrate names it in "+ + "the container AND in the migration action", out.Places) + } + if remaining := strings.Count(string(out.Bundle), out.Was); remaining != 0 { + t.Errorf("the produced bundle still names %q in %d place(s)", out.Was, remaining) + } + if got := strings.Count(string(out.Bundle), held); got != out.Places { + t.Errorf("the produced bundle names the held image %d time(s), and %d were replaced", + got, out.Places) + } +} + +// Third-party images are somebody else's, at somebody else's registry, and the installer has no +// business touching them (novox/hq ADR 0006). +func TestPostgresAndTheBrokerAreLeftExactlyAsTheyWere(t *testing.T) { + template := theRealBundle(t) + + out, err := Rewrite(template, held) + if err != nil { + t.Fatal(err) + } + + produced := containerImages(out.Declaration) + for _, id := range []string{"store", "broker"} { + image, named := produced[id] + if !named { + t.Fatalf("the substrate example no longer declares a %q container", id) + } + // Compared against the template's own text rather than against an expectation written + // here: what is being defended is "unchanged", and the template is the only thing that + // knows what it said. + if !strings.Contains(string(template), `"image": "`+image+`"`) { + t.Errorf("%s is now %q, which the template does not say", id, image) + } + if strings.HasPrefix(image, "sha256:") { + t.Errorf("%s was rewritten to an image this machine holds, and nothing holds it", id) + } + } + + // And the claim in the report is the same claim, so a person reading it is reading evidence. + if len(out.Kept) != 2 { + t.Errorf("the rewrite reports %d untouched image(s): %v", len(out.Kept), out.Kept) + } +} + +// A bundle with no control plane raises a store and a broker and no mesh. Refused, because +// applying it would succeed and leave a machine that looks bootstrapped. +func TestABundleThatNamesNoControlPlaneIsRefused(t *testing.T) { + template := []byte(`{"declaration":1,"resources":[ + {"id":"store","type":"container","name":"mesh-store","image":"postgres@sha256:` + + strings.Repeat("7", 64) + `"} + ]}`) + + _, err := Rewrite(template, held) + if err == nil { + t.Fatal("a bundle with no control plane was rewritten and would have been applied") + } + // The refusal has to be actionable: it says what the bundle DID declare, so somebody can see + // they pointed it at the wrong file or misspelled the id. + if !strings.Contains(err.Error(), ControlPlaneID) || !strings.Contains(err.Error(), "store") { + t.Errorf("the refusal names neither what was wanted nor what was there: %v", err) + } +} + +func TestAControlPlaneThatIsNotAContainerIsRefused(t *testing.T) { + template := []byte(`{"declaration":1,"resources":[ + {"id":"control-plane","type":"package","package":"mesh-control"} + ]}`) + if _, err := Rewrite(template, held); err == nil { + t.Fatal("a control plane declared as a package was accepted, and a package has no image") + } +} + +// Idempotence. This program is run over and over while somebody gets a machine working, and the +// second run must be able to say the bundle already names this image rather than reporting a +// rewrite it did not perform. +func TestRewritingABundleThatAlreadyNamesTheImageChangesNothing(t *testing.T) { + first, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + + second, err := Rewrite(first.Bundle, held) + if err != nil { + t.Fatal(err) + } + if second.Changed { + t.Error("re-running the rewrite reported a change, and the image was already the one held") + } + if string(second.Bundle) != string(first.Bundle) { + t.Error("re-running the rewrite produced different bytes") + } + if second.Was != held { + t.Errorf("the second run reports it replaced %q; it replaced nothing", second.Was) + } +} + +// A DIFFERENT image, though, must move — the ordinary case of a new control plane being installed +// over an old one. "Already correct" must not be the same code path as "already ran". +func TestANewImageReplacesAnOlderHeldOne(t *testing.T) { + first, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + second, err := Rewrite(first.Bundle, otherHeld) + if err != nil { + t.Fatal(err) + } + if !second.Changed || second.Was != held || second.Now != otherHeld { + t.Errorf("a new image did not replace the old one: changed=%v was=%q now=%q", + second.Changed, second.Was, second.Now) + } +} + +// The produced bundle is meant to be READ. Re-serialising a parsed declaration would drop every +// comment in the template, and the substrate example is mostly comments — each one recording why a +// resource is the way it is, several of them paid for in the lab. +func TestTheProducedBundleKeepsTheTemplatesComments(t *testing.T) { + template := theRealBundle(t) + out, err := Rewrite(template, held) + if err != nil { + t.Fatal(err) + } + const remembered = "NAMED VOLUME" + if !strings.Contains(string(out.Bundle), remembered) { + t.Errorf("the produced bundle lost the template's comments; %q is gone", remembered) + } +} + +// The installer must refuse its own bad input in its own words, rather than writing a bundle and +// letting the host refuse a declaration somebody did not write. +func TestSomethingThatIsNotAnImageIdIsRefused(t *testing.T) { + for _, bad := range []string{ + "", + "mesh-control:latest", + "sha256:abc", + "sha256:" + strings.Repeat("1", 63), + "sha256:" + strings.Repeat("g", 64), + "mesh-control@sha256:" + strings.Repeat("1", 64), + } { + if _, err := Rewrite(theRealBundle(t), bad); err == nil { + t.Errorf("image id %q was accepted", bad) + } + } +} + +// The address every enrolment token will carry is reported and never invented. It differs per +// machine and it is silently fatal when wrong: a node enrols against a dead address and nothing +// says so until it fails to come back. +func TestTheAddressNodesWillDialIsReportedAndNotRewritten(t *testing.T) { + template := theRealBundle(t) + out, err := Rewrite(template, held) + if err != nil { + t.Fatal(err) + } + if out.BrokerAddress == "" { + t.Fatal("the substrate example no longer says what address enrolling nodes will dial") + } + if !strings.Contains(string(out.Bundle), out.BrokerAddress) { + t.Errorf("the produced bundle no longer carries %q — it was rewritten, and nothing here "+ + "knows this machine's address", out.BrokerAddress) + } +} + +// --------------------------------------------------------------------------------------------- +// The rename, which is what makes genesis a pivot rather than a handover (novox/hq ADR 0067). +// --------------------------------------------------------------------------------------------- + +// **This is the test that dissolves the blocker.** The substrate raises a control plane and a +// module later declares one; if both are called `mesh-control` then for one moment two owners hold +// one container, and the host — which tracks what it owns — has no way to stop owning something +// without destroying it. Nothing here invents such a mechanism. The substrate's container is +// called `temp-mesh-control` instead, and there are simply two containers. +func TestTheSubstratesControlPlaneMovesOutOfTheModulesWay(t *testing.T) { + out, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + if !out.Renamed { + t.Error("the rewrite reported nothing renamed, and the template named it mesh-control") + } + if out.TempName != "temp-mesh-control" { + t.Errorf("the substrate's control plane is called %q", out.TempName) + } + control, err := controlPlaneIn(out.Declaration) + if err != nil { + t.Fatal(err) + } + if control.Name != out.TempName { + t.Errorf("the produced bundle calls it %q, want %q", control.Name, out.TempName) + } + // And the plain name is free, which is the whole point: it belongs to the module now. + for _, name := range containerNames(out.Declaration) { + if name == ControlPlaneModule { + t.Errorf("the produced bundle still declares a container called %q, which the module "+ + "will also declare", ControlPlaneModule) + } + } +} + +// The image reference contains the string `mesh-control` too, and it is not a container name. A +// substitution that caught it would produce `…/temp-mesh-control@sha256:…`, which no registry +// serves — and it would be found inside a pull rather than here. +func TestTheImageReferenceIsNotMistakenForTheContainerName(t *testing.T) { + out, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(out.Bundle), TempPrefix+"mesh-control@") || + strings.Contains(string(out.Bundle), "/"+TempPrefix+"mesh-control") { + t.Error("the rename reached inside an image reference") + } +} + +// Everything else keeps the name the substrate gave it. The store and the broker are containers +// too, and a rename that moved them would leave a machine whose substrate the host cannot find. +func TestRenamingTheControlPlaneLeavesEveryOtherContainerAlone(t *testing.T) { + before, err := declaration.ParseFileTrusted(theRealBundle(t)) + if err != nil { + t.Fatal(err) + } + out, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + was, now := containerNames(before), containerNames(out.Declaration) + for id, name := range was { + if id == ControlPlaneID { + continue + } + if now[id] != name { + t.Errorf("%s was renamed from %q to %q", id, name, now[id]) + } + } +} + +// A re-run against a bundle this installer produced renames nothing and says so. The installer is +// run over and over while somebody gets a machine working, and a step that could not tell "already +// done" from "just done" makes the second run indistinguishable from the first. +func TestRewritingABundleThisAlreadyProducedRenamesNothing(t *testing.T) { + first, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + second, err := Rewrite(first.Bundle, held) + if err != nil { + t.Fatal(err) + } + if second.Renamed { + t.Error("a bundle already naming temp-mesh-control was renamed again") + } + if second.TempName != first.TempName { + t.Errorf("the second pass calls it %q and the first called it %q", + second.TempName, first.TempName) + } + if string(second.Bundle) != string(first.Bundle) { + t.Error("rewriting a produced bundle changed it") + } +} diff --git a/internal/bootstrap/talk.go b/internal/bootstrap/talk.go new file mode 100644 index 0000000..eab5a14 --- /dev/null +++ b/internal/bootstrap/talk.go @@ -0,0 +1,119 @@ +package bootstrap + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// controlPlane is the running control plane, asked things. +// +// **Through `docker exec`, not over a network.** The control plane listens on nothing — `serve` is +// a broker consumer, and every administrative verb is a subcommand of the same binary that opens +// the stores directly (mesh-control's own usage). So the way to tell a mesh anything, from the +// machine the mesh is on, is to run its binary inside its own container. That is also what the lab +// does, and having the installer and the lab drive the mesh identically is the point: the lab is +// meant to exercise the installer, not a second procedure that resembles it. +// +// It carries which container, because the whole pivot turns on there being two of them: the +// substrate's `temp-mesh-control` for steps 6 to 9, and the module's `mesh-control` afterwards. +type controlPlane struct { + container string + run Runner + timeout time.Duration +} + +// within is the same control plane, asked with a different patience. +// +// A copy rather than a field somebody sets, so a slow command cannot leave every command after it +// slow: the caller that needs the long wait says so on the call. +func (c controlPlane) within(timeout time.Duration) controlPlane { + c.timeout = timeout + return c +} + +// tell runs a mesh-control subcommand and gives back what it said. +// +// The failure carries the command AND the output. A mesh-control refusal is a paragraph explaining +// what is wrong — "nothing provides route, wanted by registry" — and an installer that reported +// only "exit status 1" would throw away the one thing a person needs. +func (c controlPlane) tell(ctx context.Context, args ...string) (string, error) { + asking, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + + out, err := c.run(asking, "docker", append( + []string{"exec", c.container, controlPlaneBinary}, args...)...) + if err != nil { + return out, fmt.Errorf("`%s %s` was refused: %w\n%s", + c.container, strings.Join(args, " "), err, indent(strings.TrimSpace(out))) + } + return out, nil +} + +// carry puts a file inside the control plane's container. +// +// **Because every command that takes a file reads it from its own filesystem.** `module add +// ` and `secret accept --from ` open a path, and the process doing the opening is +// inside the container. The installer is not. So the file is copied in first, exactly as the lab +// does it. +// +// The image is `FROM scratch` and has no shell, so nothing inside can move a file, change its mode +// or clean up after itself. What is copied in stays until the container is replaced — which, for +// the temporary control plane, is a container that gets removed at step 10 and takes its contents +// with it. +func (c controlPlane) carry(ctx context.Context, local, remote string) error { + asking, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + + if _, err := c.run(asking, "docker", "cp", local, c.container+":"+remote); err != nil { + return fmt.Errorf("cannot put %s into %s at %s: %w", local, c.container, remote, err) + } + return nil +} + +// carrying writes bytes to a temporary file on the machine and copies them into the container. +// +// **0644, and that is not carelessness — 0600 would break it.** `docker cp` keeps the ownership and +// mode a file had outside, the control plane's image runs as 65534, and the process that reads +// these files is that one. The lab paid for this exactly once: a 0600 root-owned key copied in +// landed unreadable, `secret accept` failed with `permission denied`, and what depended on it +// crash-looped on material it never received. There is no shell in the image to chown it with. +// +// What goes through here is a module manifest and a store connection string. The connection is the +// same value the produced bundle already holds in the clear — a substrate names its own bootstrap +// credentials, and at genesis there is nowhere else for them to be — so this widens nothing. The +// file on the machine is removed at once, and the copy inside the container goes when the +// container does, which for the temporary control plane is step 10. +func (c controlPlane) carrying(ctx context.Context, name string, content []byte, remote string) error { + local := filepath.Join(os.TempDir(), name) + if err := os.WriteFile(local, content, 0o644); err != nil { + return fmt.Errorf("nowhere to stage %s before copying it into %s: %w", name, c.container, err) + } + defer os.Remove(local) + return c.carry(ctx, local, remote) +} + +func indent(s string) string { + if s == "" { + return "" + } + return " " + strings.ReplaceAll(s, "\n", "\n ") +} + +// mentions reports whether one of a listing's lines starts with this exact word. +// +// Line-and-word rather than a substring search, because these listings are columns and a +// substring match would find `registry` inside `registry-mirror` and report a module installed +// that is not. Every one of mesh-control's `list` verbs prints the name first on the line. +func mentions(listing, name string) bool { + for _, line := range strings.Split(listing, "\n") { + first, _, _ := strings.Cut(strings.TrimSpace(line), " ") + if first == name { + return true + } + } + return false +} diff --git a/internal/bootstrap/verify.go b/internal/bootstrap/verify.go new file mode 100644 index 0000000..4bed6fe --- /dev/null +++ b/internal/bootstrap/verify.go @@ -0,0 +1,167 @@ +package bootstrap + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +// controlPlaneBinary is where the control plane's program lives in its own image. +// +// A path rather than a shell command, because the image is `FROM scratch` and holds one static +// binary and nothing else — no shell to invoke, nothing to interpret a command line +// (mesh-control's Dockerfile, novox/hq ADR 0006). That is a property of the image this installer +// carries, which is why the path can be written down here. +const controlPlaneBinary = "/mesh-control" + +// answerEvery is how often the control plane is asked again while it is starting. +var answerEvery = 2 * time.Second + +// Verified is what the substrate was found to be. +type Verified struct { + // Running is every long-running container the bundle declares, confirmed up. + Running []string + // Answered is the control plane's own first words back, so the report shows the reply rather + // than asserting there was one. + Answered string +} + +// Verify proves the substrate is up and the control plane replies. +// +// **A container that is up is not a control plane that replies**, and this project has paid for +// that distinction more than once: a runtime reports a container running from the moment the +// process starts, which is before it has opened a database, before it has read its configuration, +// and before it has failed to. So the containers are checked, and then the program inside one of +// them is asked a question and has to answer it. +// +// The question is `status`, and it is chosen rather than convenient: answering it means the +// control plane opened all three of its stores from the environment the bundle gave it. A reply +// therefore proves the image runs, that `network: host` really does reach the store on this +// machine, and that the contexts' schemas migrated — the three things the steps before this were +// for. There is no HTTP endpoint to curl: `serve` is a broker consumer, not a web server. +// +// It waits. A control plane that is not answering yet and a control plane that will never answer +// look identical for the first few seconds, and refusing on the first attempt would make a correct +// bootstrap fail for being observed too early. +func Verify(ctx context.Context, d *declaration.Declaration, run Runner, probe, wait time.Duration, + say func(string)) (Verified, error) { + + var out Verified + + control, err := controlPlaneIn(d) + if err != nil { + return out, err + } + + for _, container := range longRunning(d) { + state, err := containerRunning(ctx, run, probe, container) + if err != nil { + return out, err + } + if !state.running { + return out, fmt.Errorf( + "the container %q is %s, not running.\n"+ + "The apply reported success, so it was created — what it did afterwards is "+ + "in `docker logs %s`", container, state.status, container) + } + out.Running = append(out.Running, container) + say(" running " + container) + } + + answer, err := waitForTheControlPlane(ctx, run, probe, wait, control.Name, say) + if err != nil { + return out, err + } + out.Answered = answer + return out, nil +} + +func waitForTheControlPlane(ctx context.Context, run Runner, probe, wait time.Duration, + container string, say func(string)) (string, error) { + + deadline := time.Now().Add(wait) + var last error + for { + asking, cancel := context.WithTimeout(ctx, probe) + out, err := run(asking, "docker", "exec", container, controlPlaneBinary, "status") + cancel() + + answer := strings.TrimSpace(firstLineOf(out)) + switch { + case err != nil: + last = err + case answer == "": + // Exit zero and nothing said. Treated as no answer rather than as success: a program + // that returns silence is not one that has been asked anything. + last = fmt.Errorf("it exited without saying anything") + default: + say(" replies " + container + ": " + answer) + return answer, nil + } + + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(answerEvery): + } + } + return "", fmt.Errorf( + "the container %q is running and the control plane in it does not answer, after waiting "+ + "%s: %v\n"+ + "Running is not replying. `status` opens this mesh's three stores, so what failed is "+ + "most likely the store or the schemas rather than the control plane itself — "+ + "`docker logs %s` says which", container, wait, last, container) +} + +type containerState struct { + running bool + status string +} + +func containerRunning(ctx context.Context, run Runner, probe time.Duration, name string) (containerState, error) { + asking, cancel := context.WithTimeout(ctx, probe) + defer cancel() + + // Both facts in one answer, so a container that is not running is reported with what it IS + // rather than with the absence of what it should be. + out, err := run(asking, "docker", "inspect", "--format", "{{.State.Running}} {{.State.Status}}", name) + if err != nil { + return containerState{}, fmt.Errorf( + "the container %q is not there at all, and the apply reported it applied: %w", name, err) + } + running, status, _ := strings.Cut(strings.TrimSpace(firstLineOf(out)), " ") + if status == "" { + status = "in a state the runtime did not name" + } + return containerState{running: running == "true", status: status}, nil +} + +// longRunning is every container the bundle expects to still be there afterwards. +// +// A run-once step has exited by design and a scheduled step has deliberately never been started +// (novox/hq ADR 0052, ADR 0053), so asking either of them to be running would be asking the +// substrate to be something other than what it declared. +func longRunning(d *declaration.Declaration) []string { + var names []string + for _, r := range d.Resources { + container, ok := r.(*declaration.Container) + if !ok || container.RunOnce || container.Schedule != "" { + continue + } + names = append(names, container.Name) + } + return names +} + +func firstLineOf(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} diff --git a/internal/bootstrap/verify_test.go b/internal/bootstrap/verify_test.go new file mode 100644 index 0000000..3711d7d --- /dev/null +++ b/internal/bootstrap/verify_test.go @@ -0,0 +1,170 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +func substrate(t *testing.T) *declaration.Declaration { + t.Helper() + out, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + return out.Declaration +} + +// **A container that is up is not a control plane that replies**, and this project has paid for +// that distinction more than once. A runtime reports a container running from the moment its +// process starts — before it has opened a database, and before it has failed to. +func TestAContainerThatIsUpIsNotAControlPlaneThatReplies(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + runtime := &asked{answer: func(_ string, args []string) (string, error) { + switch args[0] { + case "inspect": + return "true running\n", nil + case "exec": + // Up, and saying nothing. The program inside is not answering. + return "", errors.New("exit status 1") + } + return "", fmt.Errorf("unexpected command: %v", args) + }} + + _, err := Verify(context.Background(), substrate(t), runtime.run, + time.Second, 0, func(string) {}) + if err == nil { + t.Fatal("every container was running, nothing answered, and the substrate was reported up") + } + for _, wanted := range []string{"mesh-control", "Running is not replying", "docker logs"} { + if !strings.Contains(err.Error(), wanted) { + t.Errorf("the failure does not mention %q:\n%v", wanted, err) + } + } +} + +// Exit zero and silence is not an answer either. A program that returns nothing has not been asked +// anything, and treating it as success is the same fault one level down. +func TestAControlPlaneThatSaysNothingHasNotAnswered(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return "true running\n", nil + } + return " \n", nil + }} + + if _, err := Verify(context.Background(), substrate(t), runtime.run, + time.Second, 0, func(string) {}); err == nil { + t.Fatal("a control plane that exited zero without saying anything was accepted") + } +} + +// The substrate answering is the whole point, and what it said is reported rather than asserted. +func TestASubstrateThatIsUpAndAnsweringIsAccepted(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return "true running\n", nil + } + return "1 node, 0 waiting\n", nil + }} + + verified, err := Verify(context.Background(), substrate(t), runtime.run, + time.Second, 0, func(string) {}) + if err != nil { + t.Fatal(err) + } + // Three long-running containers: the store, the broker and the TEMPORARY control plane. The + // run-once and scheduled shapes are excluded on purpose — a step that has exited is not a + // fault. The name is `temp-mesh-control` because the permanent one is a module and takes the + // plain name (novox/hq ADR 0067), which is what makes the two of them coexist at all. + want := []string{"mesh-store", "mesh-broker", "temp-mesh-control"} + if len(verified.Running) != len(want) { + t.Fatalf("confirmed %v running, want %v", verified.Running, want) + } + for i := range want { + if verified.Running[i] != want[i] { + t.Errorf("confirmed %v running, want %v", verified.Running, want) + } + } + if verified.Answered != "1 node, 0 waiting" { + t.Errorf("the control plane's reply is reported as %q", verified.Answered) + } +} + +// A control plane that is still opening its stores is waited for, not refused. Refusing on the +// first attempt would make a correct bootstrap fail for being observed too early. +func TestAControlPlaneThatIsStillStartingIsWaitedFor(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + attempts := 0 + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return "true running\n", nil + } + attempts++ + if attempts < 3 { + return "", errors.New("exit status 1") + } + return "1 node\n", nil + }} + + if _, err := Verify(context.Background(), substrate(t), runtime.run, + time.Second, time.Second, func(string) {}); err != nil { + t.Fatalf("a control plane that answered on the third ask was refused: %v", err) + } +} + +// A container that exited is named with what it IS, so somebody can go and read its logs rather +// than being told only that something is not what it should be. +func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if args[0] == "inspect" && args[len(args)-1] == "mesh-broker" { + return "false exited\n", nil + } + if args[0] == "inspect" { + return "true running\n", nil + } + return "", fmt.Errorf("unexpected command: %v", args) + }} + + _, err := Verify(context.Background(), substrate(t), runtime.run, + time.Second, 0, func(string) {}) + if err == nil { + t.Fatal("a container that had exited was reported as part of a running substrate") + } + if !strings.Contains(err.Error(), "mesh-broker") || !strings.Contains(err.Error(), "exited") { + t.Errorf("the failure does not say which container is in what state: %v", err) + } +} + +// The control plane is asked by running the binary in its own image directly, because the image is +// `FROM scratch` and has no shell for a command line to be interpreted by. +func TestTheControlPlaneIsAskedByRunningItsOwnBinary(t *testing.T) { + runtime := &asked{answer: func(_ string, args []string) (string, error) { + if args[0] == "inspect" { + return "true running\n", nil + } + return "1 node\n", nil + }} + if _, err := Verify(context.Background(), substrate(t), runtime.run, + time.Second, 0, func(string) {}); err != nil { + t.Fatal(err) + } + if !runtime.ran("docker exec " + TempPrefix + "mesh-control " + controlPlaneBinary + " status") { + t.Errorf("the control plane was never asked anything: %v", runtime.commands) + } +} diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index 3ffa8cf..84a6824 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -874,20 +874,41 @@ func checkMode(where, mode string) []string { return nil } -// checkImage insists on a digest. +// checkImage insists on content, not on a name. // // A tag moves and a digest does not. The bundle's whole claim is that what it names is exact // (novox/hq ADR 0006), and a bundle pinning `postgres:17` pins nothing — it names whatever // that tag points at on the day the host happens to run. +// +// **Two forms say something exact, and only one of them needs a registry.** `name@sha256:…` is a +// manifest digest, which a registry assigns on push. A bare `sha256:…` is an image the machine +// already holds, addressed by the digest of its own configuration — equally immutable, equally +// unforgeable, and requiring nothing to have served it. +// +// That second form is what a first machine needs. The mesh's own control plane exists in no public +// registry and never will: it is built from source, and until this mesh has a registry of its own +// there is nowhere to push it to and therefore no manifest digest to name it by. Insisting on one +// would mean a registry has to exist before the thing that lets a mesh have a registry can start — +// which is not a pin, it is a dependency the rule accidentally created. A machine that built an +// image, or was handed one, can name it by what it is. func checkImage(where, image string) []string { if image == "" { return []string{where + ": a container needs an image"} } + // An image this machine holds, named by the digest of its own configuration. + if strings.HasPrefix(image, "sha256:") { + if len(image) != len("sha256:")+64 { + return []string{fmt.Sprintf( + "%s: image id %q is not a sha256 digest", where, image)} + } + return nil + } name, digest, found := strings.Cut(image, "@") if !found || name == "" { return []string{fmt.Sprintf( - "%s: image %q is not pinned. Write it as name@sha256:... — a tag moves, and a "+ - "bundle that pinned a tag would not be pinned", where, image)} + "%s: image %q is not pinned. Write it as name@sha256:… — or as sha256:… for an image "+ + "this machine already holds. A tag moves, and a bundle that pinned a tag would "+ + "not be pinned", where, image)} } if !strings.HasPrefix(digest, "sha256:") || len(digest) != len("sha256:")+64 { return []string{fmt.Sprintf( diff --git a/internal/declaration/declaration_test.go b/internal/declaration/declaration_test.go index 46734b3..11244bc 100644 --- a/internal/declaration/declaration_test.go +++ b/internal/declaration/declaration_test.go @@ -219,6 +219,32 @@ func TestAnImageMustBePinnedByDigest(t *testing.T) { } } +// An image the machine already holds, addressed by the digest of its own configuration. +// +// **The form a first machine needs.** A manifest digest is assigned by a registry on push, so +// insisting on one means a registry has to exist before the thing that lets a mesh have a registry +// can start. The mesh's own control plane is built from source and lives in no public registry; a +// machine that built it, or was handed it, names it by what it is — a content address, not a name, +// and immutable in exactly the way the rule asks for. +func TestAnImageTheMachineHoldsIsNamedByItsOwnDigest(t *testing.T) { + held := "sha256:" + strings.Repeat("b", 64) + if _, err := ParseTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"control","type":"container","name":"mesh-control","image":"` + held + `"} + ]}`)); err != nil { + t.Errorf("an image named by its own digest was refused: %v", err) + } + + // Still a digest, though. A truncated one names several images, and which one ran would be + // whichever the runtime happened to match first. + for _, bad := range []string{"sha256:abc", "sha256:", "sha256:" + strings.Repeat("b", 63)} { + if _, err := ParseTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"control","type":"container","name":"mesh-control","image":"` + bad + `"} + ]}`)); err == nil { + t.Errorf("image id %q was accepted and is not a digest", bad) + } + } +} + func TestAFieldTheNewTypesDoNotUseIsRefused(t *testing.T) { // The field-set check must cover the types added last, not only the three it was written // for. A package that carries a `content` is a control plane believing it asked for diff --git a/internal/image/control-plane.tar b/internal/image/control-plane.tar new file mode 100644 index 0000000..e26e4f9 --- /dev/null +++ b/internal/image/control-plane.tar @@ -0,0 +1,8 @@ +This is not a saved image. It is the placeholder that keeps this repository buildable. + +A release build replaces this file with the output of `docker save` and puts it back afterwards: + + make bootstrap IMAGE=mesh-control: + +An installer built with this file present carries no control plane, and says so in preflight +rather than getting a machine part-way to being a mesh and stopping. diff --git a/internal/image/image.go b/internal/image/image.go new file mode 100644 index 0000000..55d77c5 --- /dev/null +++ b/internal/image/image.go @@ -0,0 +1,190 @@ +// Package image is the control plane's image, carried inside the installer. +// +// **Carried rather than built, and that is not an optimisation.** The mesh's forge runs on the +// mesh. A bootstrap that needed the control plane's source in order to build it would need a +// forge to fetch that source from, and the forge is one of the things the mesh raises — so the +// mesh would be required in order to raise the mesh. Embedding the image breaks that cycle the +// same way `internal/bundle` breaks it for the declaration: the installer arrives holding +// everything a bare machine has to be given, and a bare machine is given one file +// (novox/hq ADR 0005). +// +// It is also the rule the rest of tier 0 already follows. Nobody compiles `mesh-host` on the +// machine it will run on; the binary is put there. The image it raises arrives the same way. +// +// What is embedded is the output of `docker save` — a tar holding the image's config, its layers +// and a `manifest.json` naming them. The control plane's image is `FROM scratch` with one static +// binary in it (novox/hq ADR 0006), so this is tens of megabytes rather than hundreds. +package image + +import ( + "archive/tar" + "bytes" + _ "embed" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "strings" +) + +// The saved image, replaced at release time by `make bootstrap`. +// +// What is committed here is a placeholder, for the same reason `internal/bundle` commits locks +// that are only comments: `go:embed` refuses to compile against a file that is not there, so a +// checkout with nothing embedded would not build at all — and someone reading this repository or +// running `go test ./...` would meet a compile error instead of a program. The placeholder keeps +// the tree buildable and makes the absence a thing the installer *says*, at the earliest moment +// it can, rather than a thing a compiler says to the wrong person. +// +// It is small and it is committed. A saved image is not, and `make bootstrap` puts one here for +// the length of one build and then puts the placeholder back — exactly what `make host` does with +// the bundle it embeds. +// +//go:embed control-plane.tar +var saved []byte + +// ErrEmpty means this installer carries no control-plane image. +// +// A separate error rather than a message, so the caller can refuse in preflight — before a +// machine has been touched — instead of discovering it at the load, after the runtime has been +// probed and a bundle has been written. +var ErrEmpty = errors.New( + "this mesh-bootstrap carries no control-plane image, so it cannot raise a mesh. A release " + + "build embeds one: `make bootstrap IMAGE=` in the mesh-host repository, where " + + " is a control-plane image already built from the mesh-control source") + +// IsEmpty reports whether anything was built in. +// +// It asks whether the bytes are a tar rather than comparing them against the placeholder's text, +// because the question that matters is "can this be loaded", and a truncated or corrupted embed +// answers no to that while matching no placeholder. A tar's first header carries the string +// `ustar` at offset 257 and nothing else does by accident. +func IsEmpty() bool { + const magicAt, magic = 257, "ustar" + return len(saved) < magicAt+len(magic) || string(saved[magicAt:magicAt+len(magic)]) != magic +} + +// Saved returns the embedded tar, for loading into a container runtime. +func Saved() ([]byte, error) { + if IsEmpty() { + return nil, ErrEmpty + } + return saved, nil +} + +// manifestEntry is the part of a saved image's `manifest.json` this needs. +// +// One field. The layers are the runtime's business and the repository tags are decoration — what +// is wanted is the config, because the digest of the config IS the image id. +type manifestEntry struct { + Config string `json:"Config"` + RepoTags []string `json:"RepoTags"` +} + +// ArchiveID is what this archive calls the image it holds. **It is not necessarily the id the +// runtime will assign when the archive is loaded, and it must never be what a bundle names.** +// +// This was called ID, and its comment said it was the id the runtime would give the image once +// loaded. That was wrong, and wrong in the worst available way: right on the machine the image +// was saved on, and wrong on the machine it was carried to. An image id is the sha256 of the +// image's *configuration document*, and a runtime rewrites that document as it loads — a newer +// Docker saves in one format and an older one stores it in another. Same layers, same program, +// different name. Measured on a live raise: +// +// saved on the workstation sha256:b86bb81ca2f9691f24f4725f50962d1e49c98c5ffe211113241243d42d18ceea +// loaded on the machine sha256:2dc219046c73702fc640317f0342a28ec962ef1e9ef547b2f02861c508ca78fb +// +// A bundle naming this id would then name an image the machine does not hold — and nothing serves +// an image named by the digest of its own configuration, which is the whole point of naming one +// that way, so the apply would stop at a pull that cannot succeed. The id a bundle names is read +// back from the runtime after the load (`internal/bootstrap`.Load), which is the only place it is +// a fact rather than a prediction. +// +// What it is still good for is a statement about the FILE — which build somebody embedded — and +// as a hint printed beside the runtime's answer when the two differ, so a person can see that +// they did. +// +// `manifest.json` names the configuration document by its digest — as `<64hex>.json` in the older +// layout and `blobs/sha256/<64hex>` in the OCI one — so both forms reduce to the same sixty-four +// characters. +func ArchiveID(saved []byte) (string, error) { + manifest, err := fileFromTar(saved, "manifest.json") + if err != nil { + return "", err + } + + var entries []manifestEntry + if err := json.Unmarshal(manifest, &entries); err != nil { + return "", fmt.Errorf( + "the carried image has a manifest.json this cannot read, so the image it holds "+ + "cannot be named: %w", err) + } + // One image, deliberately. A tar holding several would leave the installer choosing which + // one is the control plane, and a bootstrap must not be the thing that guesses. + if len(entries) != 1 { + return "", fmt.Errorf( + "the carried image holds %d images, and the installer raises exactly one control "+ + "plane. Save a single image: `docker save --output … `", len(entries)) + } + + digest := strings.TrimSuffix(path.Base(entries[0].Config), ".json") + if !isSHA256(digest) { + return "", fmt.Errorf( + "the carried image names its configuration %q, which is not a sha256 digest. An "+ + "image id is the digest of that configuration, so there is nothing to call this "+ + "image", entries[0].Config) + } + return "sha256:" + digest, nil +} + +// Tags is what the saved image was called when it was saved. +// +// **Not decoration any more, and this comment used to say it was.** A tag is exactly what a pinned +// bundle may not rely on (novox/hq ADR 0006) and none of this ever reaches a bundle — but the tag +// is how the installer ASKS the runtime what id it assigned, because the id itself is not +// knowable beforehand (see ArchiveID). It is the one name that survives `docker save` and +// `docker load` unchanged, which is precisely what the id does not. +// +// It also still says which build somebody embedded, which is otherwise sixty-four hex characters. +func Tags(saved []byte) []string { + manifest, err := fileFromTar(saved, "manifest.json") + if err != nil { + return nil + } + var entries []manifestEntry + if err := json.Unmarshal(manifest, &entries); err != nil || len(entries) == 0 { + return nil + } + return entries[0].RepoTags +} + +func fileFromTar(archive []byte, want string) ([]byte, error) { + reader := tar.NewReader(bytes.NewReader(archive)) + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + return nil, fmt.Errorf( + "the carried image has no %s, so it is not something `docker save` produced. "+ + "Embed the output of `docker save`, not a layer or a build context", want) + } + if err != nil { + return nil, fmt.Errorf("the carried image cannot be read as a tar: %w", err) + } + if path.Clean(header.Name) == want { + return io.ReadAll(reader) + } + } +} + +func isSHA256(s string) bool { + if len(s) != 64 { + return false + } + for _, c := range s { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} diff --git a/internal/image/image_test.go b/internal/image/image_test.go new file mode 100644 index 0000000..ef93e6a --- /dev/null +++ b/internal/image/image_test.go @@ -0,0 +1,160 @@ +package image + +import ( + "archive/tar" + "bytes" + "encoding/json" + "strings" + "testing" +) + +// Each test names the decision it defends (novox/hq ADR 0017). + +// savedImage builds what `docker save` produces, as far as this package reads it. +func savedImage(t *testing.T, files map[string]string) []byte { + t.Helper() + var buffer bytes.Buffer + writer := tar.NewWriter(&buffer) + for name, content := range files { + header := &tar.Header{Name: name, Mode: 0o644, Size: int64(len(content))} + if err := writer.WriteHeader(header); err != nil { + t.Fatalf("building the fixture: %v", err) + } + if _, err := writer.Write([]byte(content)); err != nil { + t.Fatalf("building the fixture: %v", err) + } + } + if err := writer.Close(); err != nil { + t.Fatalf("building the fixture: %v", err) + } + return buffer.Bytes() +} + +func manifest(t *testing.T, config string, tags ...string) string { + t.Helper() + raw, err := json.Marshal([]manifestEntry{{Config: config, RepoTags: tags}}) + if err != nil { + t.Fatalf("building the fixture: %v", err) + } + return string(raw) +} + +// The archive's own id is read from the FILE, in both layouts `docker save` has used. +// +// **This test used to say that reading it here was what made the load idempotent — that knowing +// the id in advance let the installer ask "do you already hold exactly this". That was wrong.** An +// id is the digest of the image's configuration document, and a runtime rewrites that document as +// it loads, so this is a fact about the archive and not a prediction about any machine. What the +// installer asks a runtime by is the TAG, and what a bundle names is the answer the runtime gives +// back (`internal/bootstrap`.Load). +// +// It is still read and still checked, because it is what says which build somebody embedded — and +// because printing it beside the runtime's answer is how a person sees that the two differ. +func TestTheArchivesOwnIdIsReadFromTheSavedFile(t *testing.T) { + digest := strings.Repeat("a", 64) + // Both layouts `docker save` has used. The older one names the config `.json`; the OCI + // one names it `blobs/sha256/`. They carry the same sixty-four characters, and a + // reader that understood only one would work until somebody upgraded their runtime. + for _, config := range []string{digest + ".json", "blobs/sha256/" + digest} { + saved := savedImage(t, map[string]string{"manifest.json": manifest(t, config)}) + id, err := ArchiveID(saved) + if err != nil { + t.Fatalf("config %q: %v", config, err) + } + if id != "sha256:"+digest { + t.Errorf("config %q gave id %q, want sha256:%s", config, id, digest) + } + } +} + +// The tag is read, and it is not decoration: it is the name the installer asks a runtime by, +// because it is the one thing that survives `docker save` and `docker load` unchanged. The id +// does not. +func TestTheSavedTagsAreRead(t *testing.T) { + saved := savedImage(t, map[string]string{ + "manifest.json": manifest(t, strings.Repeat("b", 64)+".json", "mesh-control:v1"), + }) + got := Tags(saved) + if len(got) != 1 || got[0] != "mesh-control:v1" { + t.Errorf("tags = %v, want [mesh-control:v1]", got) + } +} + +// A tar that is not a saved image is refused with what is wrong, not with a nil id. +// +// Nothing here reaches a bundle any more, but this is still the earliest moment somebody can be +// told they embedded the wrong file — and the alternative is finding out at the load, from a +// container runtime, in a sentence about a tar rather than about what was built. +func TestSomethingThatIsNotASavedImageIsRefused(t *testing.T) { + notAnImage := savedImage(t, map[string]string{"hello": "world"}) + if _, err := ArchiveID(notAnImage); err == nil { + t.Error("a tar with no manifest.json was accepted as a saved image") + } else if !strings.Contains(err.Error(), "docker save") { + t.Errorf("the refusal does not say what to embed instead: %v", err) + } + + if _, err := ArchiveID([]byte("this is not a tar at all")); err == nil { + t.Error("bytes that are not a tar were accepted") + } +} + +// Exactly one image. A bootstrap that chose between several would be the thing that guesses which +// one is the control plane, and it would guess right until the day somebody saved two. +func TestATarHoldingSeveralImagesIsRefused(t *testing.T) { + entries, err := json.Marshal([]manifestEntry{ + {Config: strings.Repeat("a", 64) + ".json"}, + {Config: strings.Repeat("b", 64) + ".json"}, + }) + if err != nil { + t.Fatal(err) + } + saved := savedImage(t, map[string]string{"manifest.json": string(entries)}) + if _, err := ArchiveID(saved); err == nil { + t.Error("a tar holding two images was accepted") + } +} + +// An id is a digest or it is nothing. A truncated one names several images, and which one ran +// would be whichever the runtime matched first — the same reasoning `internal/declaration` gives +// for refusing a short image reference. +func TestAConfigThatIsNotADigestIsRefused(t *testing.T) { + for _, config := range []string{"config.json", "abc.json", "blobs/sha256/" + strings.Repeat("a", 63)} { + saved := savedImage(t, map[string]string{"manifest.json": manifest(t, config)}) + if _, err := ArchiveID(saved); err == nil { + t.Errorf("config %q was accepted and is not a digest", config) + } + } +} + +// The committed placeholder must read as "carries nothing", so an installer built from a plain +// checkout says so in preflight rather than getting a machine as far as a running store and +// stopping. This is the same guarantee `internal/bundle` makes about a lock file of only comments. +func TestAnInstallerBuiltFromAPlainCheckoutCarriesNothing(t *testing.T) { + if !IsEmpty() { + // Not a failure of this checkout: `make bootstrap` embeds a real image and puts the + // placeholder back, so a real image here means a build was interrupted. + t.Skip("this checkout has a saved image embedded, so there is no placeholder to check") + } + if _, err := Saved(); err == nil { + t.Fatal("an installer carrying only the placeholder reported it carries an image") + } +} + +// And "empty" is decided by whether the bytes could be loaded, not by matching the placeholder's +// text. A truncated or corrupted embed is equally unloadable and equally worth refusing early. +func TestEmptyMeansUnloadableRatherThanEqualToThePlaceholder(t *testing.T) { + restore := saved + defer func() { saved = restore }() + + saved = []byte("half a tar, cut off") + if !IsEmpty() { + t.Error("bytes that are not a tar were reported as a carried image") + } + + saved = savedImage(t, map[string]string{ + "manifest.json": manifest(t, strings.Repeat("c", 64)+".json"), + }) + if IsEmpty() { + t.Error("a real saved image was reported as no image at all") + } +}